@rubric-protocol/attest-index 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/backfill.d.ts +12 -0
- package/dist/backfill.d.ts.map +1 -0
- package/dist/backfill.js +92 -0
- package/dist/backfill.js.map +1 -0
- package/dist/cli/backfill.d.ts +3 -0
- package/dist/cli/backfill.d.ts.map +1 -0
- package/dist/cli/backfill.js +17 -0
- package/dist/cli/backfill.js.map +1 -0
- package/dist/index.d.ts +12 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +12 -0
- package/dist/index.js.map +1 -0
- package/dist/schema.d.ts +36 -0
- package/dist/schema.d.ts.map +1 -0
- package/dist/schema.js +55 -0
- package/dist/schema.js.map +1 -0
- package/dist/shard.d.ts +42 -0
- package/dist/shard.d.ts.map +1 -0
- package/dist/shard.js +100 -0
- package/dist/shard.js.map +1 -0
- package/dist/store.d.ts +38 -0
- package/dist/store.d.ts.map +1 -0
- package/dist/store.js +147 -0
- package/dist/store.js.map +1 -0
- package/package.json +35 -0
- package/src/backfill.ts +98 -0
- package/src/cli/backfill.ts +18 -0
- package/src/index.ts +19 -0
- package/src/schema.ts +87 -0
- package/src/shard.ts +121 -0
- package/src/store.ts +160 -0
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
export interface BackfillResult {
|
|
2
|
+
bundleFiles: number;
|
|
3
|
+
rowsWritten: number;
|
|
4
|
+
/** Bundles that parsed and validated but failed to insert (e.g. a conflicting duplicate decisionId). */
|
|
5
|
+
failed: number;
|
|
6
|
+
/** Files that were not valid bundles (parse error, wrong shape, or bad ts). */
|
|
7
|
+
skipped: number;
|
|
8
|
+
days: string[];
|
|
9
|
+
}
|
|
10
|
+
/** Backfill (or rebuild) the index at `indexDir` from bundles under `storeDir`. */
|
|
11
|
+
export declare function backfill(storeDir: string, indexDir: string): BackfillResult;
|
|
12
|
+
//# sourceMappingURL=backfill.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"backfill.d.ts","sourceRoot":"","sources":["../src/backfill.ts"],"names":[],"mappings":"AAaA,MAAM,WAAW,cAAc;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,wGAAwG;IACxG,MAAM,EAAE,MAAM,CAAC;IACf,+EAA+E;IAC/E,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,EAAE,CAAC;CAChB;AAqCD,mFAAmF;AACnF,wBAAgB,QAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,cAAc,CAsC3E"}
|
package/dist/backfill.js
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Backfill the index from a bundle-store directory (tasks/P2.md).
|
|
3
|
+
*
|
|
4
|
+
* Walks the store for `*.json` bundles and upserts a row per bundle. Idempotent
|
|
5
|
+
* (upsert keyed on attestationId), so re-running is a no-op and the index is
|
|
6
|
+
* fully rebuildable from the bundles alone. `bundlePath` is stored relative to
|
|
7
|
+
* the store root so the index stays portable.
|
|
8
|
+
*/
|
|
9
|
+
import { readdirSync, readFileSync } from "node:fs";
|
|
10
|
+
import { join, relative } from "node:path";
|
|
11
|
+
import { Index } from "./store.js";
|
|
12
|
+
import { rowFromBundle, shardKeyForTs } from "./schema.js";
|
|
13
|
+
function walkJson(root) {
|
|
14
|
+
const out = [];
|
|
15
|
+
const stack = [root];
|
|
16
|
+
while (stack.length > 0) {
|
|
17
|
+
const dir = stack.pop();
|
|
18
|
+
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
|
19
|
+
const full = join(dir, entry.name);
|
|
20
|
+
if (entry.isDirectory())
|
|
21
|
+
stack.push(full);
|
|
22
|
+
else if (entry.isFile() && entry.name.endsWith(".json"))
|
|
23
|
+
out.push(full);
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
out.sort();
|
|
27
|
+
return out;
|
|
28
|
+
}
|
|
29
|
+
function isBundle(value) {
|
|
30
|
+
if (typeof value !== "object" || value === null)
|
|
31
|
+
return false;
|
|
32
|
+
const b = value;
|
|
33
|
+
if (typeof b.attestationId !== "string" || b.attestationId.length === 0)
|
|
34
|
+
return false;
|
|
35
|
+
const d = b.dar;
|
|
36
|
+
if (!d || typeof d !== "object")
|
|
37
|
+
return false;
|
|
38
|
+
// Validate EVERY column the index requires as NOT NULL, so a malformed bundle
|
|
39
|
+
// is skipped up front rather than throwing a NOT NULL error mid-insert.
|
|
40
|
+
const str = (x) => typeof x === "string";
|
|
41
|
+
return (str(d.decisionId) &&
|
|
42
|
+
str(d.agentId) &&
|
|
43
|
+
str(d.ts) &&
|
|
44
|
+
str(d.schemaHash) &&
|
|
45
|
+
str(d.decisionHash) &&
|
|
46
|
+
str(d.leafType) &&
|
|
47
|
+
(d.prev === null || str(d.prev)));
|
|
48
|
+
}
|
|
49
|
+
/** Backfill (or rebuild) the index at `indexDir` from bundles under `storeDir`. */
|
|
50
|
+
export function backfill(storeDir, indexDir) {
|
|
51
|
+
const index = new Index(indexDir);
|
|
52
|
+
try {
|
|
53
|
+
const files = walkJson(storeDir);
|
|
54
|
+
const rows = [];
|
|
55
|
+
let skipped = 0;
|
|
56
|
+
for (const file of files) {
|
|
57
|
+
let parsed;
|
|
58
|
+
try {
|
|
59
|
+
parsed = JSON.parse(readFileSync(file, "utf8"));
|
|
60
|
+
}
|
|
61
|
+
catch {
|
|
62
|
+
skipped++;
|
|
63
|
+
continue;
|
|
64
|
+
}
|
|
65
|
+
if (!isBundle(parsed)) {
|
|
66
|
+
skipped++;
|
|
67
|
+
continue;
|
|
68
|
+
}
|
|
69
|
+
// Reject a ts that has no valid day key here, so grouping can't throw.
|
|
70
|
+
try {
|
|
71
|
+
shardKeyForTs(parsed.dar.ts);
|
|
72
|
+
}
|
|
73
|
+
catch {
|
|
74
|
+
skipped++;
|
|
75
|
+
continue;
|
|
76
|
+
}
|
|
77
|
+
rows.push(rowFromBundle(parsed, relative(storeDir, file)));
|
|
78
|
+
}
|
|
79
|
+
const { written, failed } = index.writeBatchResilient(rows);
|
|
80
|
+
return {
|
|
81
|
+
bundleFiles: files.length,
|
|
82
|
+
rowsWritten: written,
|
|
83
|
+
failed,
|
|
84
|
+
skipped,
|
|
85
|
+
days: index.shardDays(),
|
|
86
|
+
};
|
|
87
|
+
}
|
|
88
|
+
finally {
|
|
89
|
+
index.close();
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
//# sourceMappingURL=backfill.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"backfill.js","sourceRoot":"","sources":["../src/backfill.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACpD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAC3C,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,EAAE,aAAa,EAAE,aAAa,EAAyC,MAAM,aAAa,CAAC;AAYlG,SAAS,QAAQ,CAAC,IAAY;IAC5B,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,MAAM,KAAK,GAAG,CAAC,IAAI,CAAC,CAAC;IACrB,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,EAAG,CAAC;QACzB,KAAK,MAAM,KAAK,IAAI,WAAW,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;YAC9D,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YACnC,IAAI,KAAK,CAAC,WAAW,EAAE;gBAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;iBACrC,IAAI,KAAK,CAAC,MAAM,EAAE,IAAI,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC;gBAAE,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1E,CAAC;IACH,CAAC;IACD,GAAG,CAAC,IAAI,EAAE,CAAC;IACX,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,QAAQ,CAAC,KAAc;IAC9B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,KAAK,CAAC;IAC9D,MAAM,CAAC,GAAG,KAAgC,CAAC;IAC3C,IAAI,OAAO,CAAC,CAAC,aAAa,KAAK,QAAQ,IAAI,CAAC,CAAC,aAAa,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACtF,MAAM,CAAC,GAAG,CAAC,CAAC,GAA0C,CAAC;IACvD,IAAI,CAAC,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC9C,8EAA8E;IAC9E,wEAAwE;IACxE,MAAM,GAAG,GAAG,CAAC,CAAU,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC;IAClD,OAAO,CACL,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC;QACjB,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC;QACd,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QACT,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC;QACjB,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC;QACnB,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC;QACf,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,IAAI,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CACjC,CAAC;AACJ,CAAC;AAED,mFAAmF;AACnF,MAAM,UAAU,QAAQ,CAAC,QAAgB,EAAE,QAAgB;IACzD,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,QAAQ,CAAC,CAAC;IAClC,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACjC,MAAM,IAAI,GAAe,EAAE,CAAC;QAC5B,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,MAAe,CAAC;YACpB,IAAI,CAAC;gBACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;YAClD,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,EAAE,CAAC;gBACV,SAAS;YACX,CAAC;YACD,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;gBACtB,OAAO,EAAE,CAAC;gBACV,SAAS;YACX,CAAC;YACD,uEAAuE;YACvE,IAAI,CAAC;gBACH,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAC/B,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,EAAE,CAAC;gBACV,SAAS;YACX,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,QAAQ,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;QAC7D,CAAC;QACD,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,KAAK,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC;QAC5D,OAAO;YACL,WAAW,EAAE,KAAK,CAAC,MAAM;YACzB,WAAW,EAAE,OAAO;YACpB,MAAM;YACN,OAAO;YACP,IAAI,EAAE,KAAK,CAAC,SAAS,EAAE;SACxB,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,KAAK,CAAC,KAAK,EAAE,CAAC;IAChB,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"backfill.d.ts","sourceRoot":"","sources":["../../src/cli/backfill.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* CLI: rebuild the attestation index from a bundle store.
|
|
4
|
+
* rubric-index-backfill <bundle-store-dir> <index-dir>
|
|
5
|
+
*/
|
|
6
|
+
import { backfill } from "../backfill.js";
|
|
7
|
+
function main(argv) {
|
|
8
|
+
const [storeDir, indexDir] = argv;
|
|
9
|
+
if (!storeDir || !indexDir) {
|
|
10
|
+
process.stderr.write("usage: rubric-index-backfill <bundle-store-dir> <index-dir>\n");
|
|
11
|
+
process.exit(2);
|
|
12
|
+
}
|
|
13
|
+
const result = backfill(storeDir, indexDir);
|
|
14
|
+
process.stdout.write(JSON.stringify(result, null, 2) + "\n");
|
|
15
|
+
}
|
|
16
|
+
main(process.argv.slice(2));
|
|
17
|
+
//# sourceMappingURL=backfill.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"backfill.js","sourceRoot":"","sources":["../../src/cli/backfill.ts"],"names":[],"mappings":";AACA;;;GAGG;AACH,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAE1C,SAAS,IAAI,CAAC,IAAc;IAC1B,MAAM,CAAC,QAAQ,EAAE,QAAQ,CAAC,GAAG,IAAI,CAAC;IAClC,IAAI,CAAC,QAAQ,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC3B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,+DAA+D,CAAC,CAAC;QACtF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IACD,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAC5C,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;AAC/D,CAAC;AAED,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC"}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @rubric-protocol/attest-index — SQLite day-shard attestation index (tasks/P2.md).
|
|
3
|
+
*
|
|
4
|
+
* Writer + query lib over per-day WAL shards, plus a backfill that rebuilds the
|
|
5
|
+
* index from a bundle store. The bundle store is the source of truth; the index
|
|
6
|
+
* is a rebuildable cache.
|
|
7
|
+
*/
|
|
8
|
+
export { type IndexRow, type AttestationBundle, rowFromBundle, shardKeyForTs, shardFileName, dayKeyFromFileName, DDL, } from "./schema.js";
|
|
9
|
+
export { Shard, type ShardOptions } from "./shard.js";
|
|
10
|
+
export { Index, type IndexOptions } from "./store.js";
|
|
11
|
+
export { backfill, type BackfillResult } from "./backfill.js";
|
|
12
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,OAAO,EACL,KAAK,QAAQ,EACb,KAAK,iBAAiB,EACtB,aAAa,EACb,aAAa,EACb,aAAa,EACb,kBAAkB,EAClB,GAAG,GACJ,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,KAAK,EAAE,KAAK,YAAY,EAAE,MAAM,YAAY,CAAC;AACtD,OAAO,EAAE,KAAK,EAAE,KAAK,YAAY,EAAE,MAAM,YAAY,CAAC;AACtD,OAAO,EAAE,QAAQ,EAAE,KAAK,cAAc,EAAE,MAAM,eAAe,CAAC"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @rubric-protocol/attest-index — SQLite day-shard attestation index (tasks/P2.md).
|
|
3
|
+
*
|
|
4
|
+
* Writer + query lib over per-day WAL shards, plus a backfill that rebuilds the
|
|
5
|
+
* index from a bundle store. The bundle store is the source of truth; the index
|
|
6
|
+
* is a rebuildable cache.
|
|
7
|
+
*/
|
|
8
|
+
export { rowFromBundle, shardKeyForTs, shardFileName, dayKeyFromFileName, DDL, } from "./schema.js";
|
|
9
|
+
export { Shard } from "./shard.js";
|
|
10
|
+
export { Index } from "./store.js";
|
|
11
|
+
export { backfill } from "./backfill.js";
|
|
12
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,OAAO,EAGL,aAAa,EACb,aAAa,EACb,aAAa,EACb,kBAAkB,EAClB,GAAG,GACJ,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,KAAK,EAAqB,MAAM,YAAY,CAAC;AACtD,OAAO,EAAE,KAAK,EAAqB,MAAM,YAAY,CAAC;AACtD,OAAO,EAAE,QAAQ,EAAuB,MAAM,eAAe,CAAC"}
|
package/dist/schema.d.ts
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Attestation index schema (tasks/P2.md). One row per attested leaf; the row
|
|
3
|
+
* mirrors the DAR core identity plus the service `attestationId` and a pointer
|
|
4
|
+
* back to the bundle it was derived from (so the index is a cache, and the
|
|
5
|
+
* bundle store remains the source of truth — the index is rebuildable).
|
|
6
|
+
*/
|
|
7
|
+
import type { DarCore } from "@rubric-protocol/attest-decision";
|
|
8
|
+
export interface IndexRow {
|
|
9
|
+
attestationId: string;
|
|
10
|
+
decisionId: string;
|
|
11
|
+
agentId: string;
|
|
12
|
+
schemaHash: string;
|
|
13
|
+
decisionHash: string;
|
|
14
|
+
prev: string | null;
|
|
15
|
+
ts: string;
|
|
16
|
+
leafType: string;
|
|
17
|
+
bundlePath: string;
|
|
18
|
+
}
|
|
19
|
+
/** A stored attestation bundle: the DAR core plus its service `attestationId`. */
|
|
20
|
+
export interface AttestationBundle {
|
|
21
|
+
attestationId: string;
|
|
22
|
+
dar: DarCore;
|
|
23
|
+
[k: string]: unknown;
|
|
24
|
+
}
|
|
25
|
+
/** Derive an index row from a bundle and the path it was read from. */
|
|
26
|
+
export declare function rowFromBundle(bundle: AttestationBundle, bundlePath: string): IndexRow;
|
|
27
|
+
/** UTC day key (YYYY-MM-DD) that a row's `ts` shards into. */
|
|
28
|
+
export declare function shardKeyForTs(ts: string): string;
|
|
29
|
+
export declare const SHARD_FILE_PREFIX = "attest-";
|
|
30
|
+
export declare const SHARD_FILE_SUFFIX = ".sqlite";
|
|
31
|
+
export declare function shardFileName(dayKey: string): string;
|
|
32
|
+
/** Extract the day key from a shard file name, or null if it is not a shard. */
|
|
33
|
+
export declare function dayKeyFromFileName(name: string): string | null;
|
|
34
|
+
/** DDL. Indexes back the four queries in tasks/P2.md. */
|
|
35
|
+
export declare const DDL = "\nCREATE TABLE IF NOT EXISTS attestations (\n attestationId TEXT PRIMARY KEY,\n decisionId TEXT NOT NULL,\n agentId TEXT NOT NULL,\n schemaHash TEXT NOT NULL,\n decisionHash TEXT NOT NULL,\n prev TEXT,\n ts TEXT NOT NULL,\n leafType TEXT NOT NULL,\n bundlePath TEXT NOT NULL\n);\nCREATE UNIQUE INDEX IF NOT EXISTS ux_decisionId ON attestations(decisionId);\nCREATE INDEX IF NOT EXISTS ix_agent_ts ON attestations(agentId, ts, decisionId);\nCREATE INDEX IF NOT EXISTS ix_agent_schema ON attestations(agentId, schemaHash, ts, decisionId);\nCREATE INDEX IF NOT EXISTS ix_agent_decisionId ON attestations(agentId, decisionId);\nCREATE INDEX IF NOT EXISTS ix_agent_prev ON attestations(agentId, prev);\n";
|
|
36
|
+
//# sourceMappingURL=schema.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,kCAAkC,CAAC;AAEhE,MAAM,WAAW,QAAQ;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,kFAAkF;AAClF,MAAM,WAAW,iBAAiB;IAChC,aAAa,EAAE,MAAM,CAAC;IACtB,GAAG,EAAE,OAAO,CAAC;IAGb,CAAC,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;CACtB;AAED,uEAAuE;AACvE,wBAAgB,aAAa,CAAC,MAAM,EAAE,iBAAiB,EAAE,UAAU,EAAE,MAAM,GAAG,QAAQ,CAarF;AAED,8DAA8D;AAC9D,wBAAgB,aAAa,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,CAMhD;AAED,eAAO,MAAM,iBAAiB,YAAY,CAAC;AAC3C,eAAO,MAAM,iBAAiB,YAAY,CAAC;AAE3C,wBAAgB,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAEpD;AAED,gFAAgF;AAChF,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAI9D;AAED,yDAAyD;AACzD,eAAO,MAAM,GAAG,qyBAiBf,CAAC"}
|
package/dist/schema.js
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
/** Derive an index row from a bundle and the path it was read from. */
|
|
2
|
+
export function rowFromBundle(bundle, bundlePath) {
|
|
3
|
+
const d = bundle.dar;
|
|
4
|
+
return {
|
|
5
|
+
attestationId: bundle.attestationId,
|
|
6
|
+
decisionId: d.decisionId,
|
|
7
|
+
agentId: d.agentId,
|
|
8
|
+
schemaHash: d.schemaHash,
|
|
9
|
+
decisionHash: d.decisionHash,
|
|
10
|
+
prev: d.prev,
|
|
11
|
+
ts: d.ts,
|
|
12
|
+
leafType: d.leafType,
|
|
13
|
+
bundlePath,
|
|
14
|
+
};
|
|
15
|
+
}
|
|
16
|
+
/** UTC day key (YYYY-MM-DD) that a row's `ts` shards into. */
|
|
17
|
+
export function shardKeyForTs(ts) {
|
|
18
|
+
const key = ts.slice(0, 10);
|
|
19
|
+
if (!/^\d{4}-\d{2}-\d{2}$/.test(key)) {
|
|
20
|
+
throw new Error(`index: cannot derive shard day from ts '${ts}'`);
|
|
21
|
+
}
|
|
22
|
+
return key;
|
|
23
|
+
}
|
|
24
|
+
export const SHARD_FILE_PREFIX = "attest-";
|
|
25
|
+
export const SHARD_FILE_SUFFIX = ".sqlite";
|
|
26
|
+
export function shardFileName(dayKey) {
|
|
27
|
+
return `${SHARD_FILE_PREFIX}${dayKey}${SHARD_FILE_SUFFIX}`;
|
|
28
|
+
}
|
|
29
|
+
/** Extract the day key from a shard file name, or null if it is not a shard. */
|
|
30
|
+
export function dayKeyFromFileName(name) {
|
|
31
|
+
if (!name.startsWith(SHARD_FILE_PREFIX) || !name.endsWith(SHARD_FILE_SUFFIX))
|
|
32
|
+
return null;
|
|
33
|
+
const key = name.slice(SHARD_FILE_PREFIX.length, name.length - SHARD_FILE_SUFFIX.length);
|
|
34
|
+
return /^\d{4}-\d{2}-\d{2}$/.test(key) ? key : null;
|
|
35
|
+
}
|
|
36
|
+
/** DDL. Indexes back the four queries in tasks/P2.md. */
|
|
37
|
+
export const DDL = `
|
|
38
|
+
CREATE TABLE IF NOT EXISTS attestations (
|
|
39
|
+
attestationId TEXT PRIMARY KEY,
|
|
40
|
+
decisionId TEXT NOT NULL,
|
|
41
|
+
agentId TEXT NOT NULL,
|
|
42
|
+
schemaHash TEXT NOT NULL,
|
|
43
|
+
decisionHash TEXT NOT NULL,
|
|
44
|
+
prev TEXT,
|
|
45
|
+
ts TEXT NOT NULL,
|
|
46
|
+
leafType TEXT NOT NULL,
|
|
47
|
+
bundlePath TEXT NOT NULL
|
|
48
|
+
);
|
|
49
|
+
CREATE UNIQUE INDEX IF NOT EXISTS ux_decisionId ON attestations(decisionId);
|
|
50
|
+
CREATE INDEX IF NOT EXISTS ix_agent_ts ON attestations(agentId, ts, decisionId);
|
|
51
|
+
CREATE INDEX IF NOT EXISTS ix_agent_schema ON attestations(agentId, schemaHash, ts, decisionId);
|
|
52
|
+
CREATE INDEX IF NOT EXISTS ix_agent_decisionId ON attestations(agentId, decisionId);
|
|
53
|
+
CREATE INDEX IF NOT EXISTS ix_agent_prev ON attestations(agentId, prev);
|
|
54
|
+
`;
|
|
55
|
+
//# sourceMappingURL=schema.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"schema.js","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AA6BA,uEAAuE;AACvE,MAAM,UAAU,aAAa,CAAC,MAAyB,EAAE,UAAkB;IACzE,MAAM,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC;IACrB,OAAO;QACL,aAAa,EAAE,MAAM,CAAC,aAAa;QACnC,UAAU,EAAE,CAAC,CAAC,UAAU;QACxB,OAAO,EAAE,CAAC,CAAC,OAAO;QAClB,UAAU,EAAE,CAAC,CAAC,UAAU;QACxB,YAAY,EAAE,CAAC,CAAC,YAAY;QAC5B,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,EAAE,EAAE,CAAC,CAAC,EAAE;QACR,QAAQ,EAAE,CAAC,CAAC,QAAQ;QACpB,UAAU;KACX,CAAC;AACJ,CAAC;AAED,8DAA8D;AAC9D,MAAM,UAAU,aAAa,CAAC,EAAU;IACtC,MAAM,GAAG,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC5B,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QACrC,MAAM,IAAI,KAAK,CAAC,2CAA2C,EAAE,GAAG,CAAC,CAAC;IACpE,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,CAAC,MAAM,iBAAiB,GAAG,SAAS,CAAC;AAC3C,MAAM,CAAC,MAAM,iBAAiB,GAAG,SAAS,CAAC;AAE3C,MAAM,UAAU,aAAa,CAAC,MAAc;IAC1C,OAAO,GAAG,iBAAiB,GAAG,MAAM,GAAG,iBAAiB,EAAE,CAAC;AAC7D,CAAC;AAED,gFAAgF;AAChF,MAAM,UAAU,kBAAkB,CAAC,IAAY;IAC7C,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,iBAAiB,CAAC;QAAE,OAAO,IAAI,CAAC;IAC1F,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC;IACzF,OAAO,qBAAqB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;AACtD,CAAC;AAED,yDAAyD;AACzD,MAAM,CAAC,MAAM,GAAG,GAAG;;;;;;;;;;;;;;;;;CAiBlB,CAAC"}
|
package/dist/shard.d.ts
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A single day-shard: one WAL-mode SQLite file with the four indexed queries
|
|
3
|
+
* from tasks/P2.md. `insert` is idempotent (keyed on attestationId), which is
|
|
4
|
+
* what makes backfill idempotent and the index rebuildable from bundles alone.
|
|
5
|
+
*/
|
|
6
|
+
import Database from "better-sqlite3";
|
|
7
|
+
import { type IndexRow } from "./schema.js";
|
|
8
|
+
export interface ShardOptions {
|
|
9
|
+
readonly?: boolean;
|
|
10
|
+
}
|
|
11
|
+
export declare class Shard {
|
|
12
|
+
readonly db: Database.Database;
|
|
13
|
+
private readonly insertStmt;
|
|
14
|
+
private readonly byDecisionStmt;
|
|
15
|
+
private readonly byAgentRangeStmt;
|
|
16
|
+
private readonly byAgentSchemaStmt;
|
|
17
|
+
private readonly chainHeadStmt;
|
|
18
|
+
private readonly insertMany;
|
|
19
|
+
constructor(path: string, options?: ShardOptions);
|
|
20
|
+
/** Idempotent single-row upsert. */
|
|
21
|
+
insert(row: IndexRow): void;
|
|
22
|
+
/** Idempotent bulk upsert in one transaction. Throws on any row error. */
|
|
23
|
+
insertBatch(rows: IndexRow[]): number;
|
|
24
|
+
/**
|
|
25
|
+
* Idempotent bulk upsert with per-row error isolation, in one transaction.
|
|
26
|
+
* A row that violates a constraint (e.g. a conflicting duplicate decisionId)
|
|
27
|
+
* is skipped and counted in `failed` — the good rows still commit. This is
|
|
28
|
+
* what keeps backfill robust: one bad bundle cannot roll back a whole shard.
|
|
29
|
+
*/
|
|
30
|
+
insertResilient(rows: IndexRow[]): {
|
|
31
|
+
written: number;
|
|
32
|
+
failed: number;
|
|
33
|
+
};
|
|
34
|
+
byDecisionId(decisionId: string): IndexRow | undefined;
|
|
35
|
+
byAgentRange(agentId: string, fromTs: string, toTs: string, limit?: number): IndexRow[];
|
|
36
|
+
byAgentSchema(agentId: string, schemaHash: string, limit?: number): IndexRow[];
|
|
37
|
+
/** The chain tip for an agent: the latest decisionId (ULID is chronological). */
|
|
38
|
+
chainHead(agentId: string): IndexRow | undefined;
|
|
39
|
+
count(): number;
|
|
40
|
+
close(): void;
|
|
41
|
+
}
|
|
42
|
+
//# sourceMappingURL=shard.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"shard.d.ts","sourceRoot":"","sources":["../src/shard.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AACtC,OAAO,EAAO,KAAK,QAAQ,EAAE,MAAM,aAAa,CAAC;AAEjD,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;AAkBD,qBAAa,KAAK;IAChB,QAAQ,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,CAAC;IAC/B,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAiC;IAC5D,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAqB;IACpD,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAqB;IACtD,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAqB;IACvD,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAqB;IACnD,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAqD;gBAEpE,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,YAAiB;IA0BpD,oCAAoC;IACpC,MAAM,CAAC,GAAG,EAAE,QAAQ,GAAG,IAAI;IAI3B,0EAA0E;IAC1E,WAAW,CAAC,IAAI,EAAE,QAAQ,EAAE,GAAG,MAAM;IAIrC;;;;;OAKG;IACH,eAAe,CAAC,IAAI,EAAE,QAAQ,EAAE,GAAG;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE;IAiBtE,YAAY,CAAC,UAAU,EAAE,MAAM,GAAG,QAAQ,GAAG,SAAS;IAItD,YAAY,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,SAAK,GAAG,QAAQ,EAAE;IAInF,aAAa,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,KAAK,SAAK,GAAG,QAAQ,EAAE;IAI1E,iFAAiF;IACjF,SAAS,CAAC,OAAO,EAAE,MAAM,GAAG,QAAQ,GAAG,SAAS;IAIhD,KAAK,IAAI,MAAM;IAIf,KAAK,IAAI,IAAI;CAGd"}
|
package/dist/shard.js
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A single day-shard: one WAL-mode SQLite file with the four indexed queries
|
|
3
|
+
* from tasks/P2.md. `insert` is idempotent (keyed on attestationId), which is
|
|
4
|
+
* what makes backfill idempotent and the index rebuildable from bundles alone.
|
|
5
|
+
*/
|
|
6
|
+
import Database from "better-sqlite3";
|
|
7
|
+
import { DDL } from "./schema.js";
|
|
8
|
+
const INSERT_SQL = `
|
|
9
|
+
INSERT INTO attestations
|
|
10
|
+
(attestationId, decisionId, agentId, schemaHash, decisionHash, prev, ts, leafType, bundlePath)
|
|
11
|
+
VALUES
|
|
12
|
+
(@attestationId, @decisionId, @agentId, @schemaHash, @decisionHash, @prev, @ts, @leafType, @bundlePath)
|
|
13
|
+
ON CONFLICT(attestationId) DO UPDATE SET
|
|
14
|
+
decisionId = excluded.decisionId,
|
|
15
|
+
agentId = excluded.agentId,
|
|
16
|
+
schemaHash = excluded.schemaHash,
|
|
17
|
+
decisionHash = excluded.decisionHash,
|
|
18
|
+
prev = excluded.prev,
|
|
19
|
+
ts = excluded.ts,
|
|
20
|
+
leafType = excluded.leafType,
|
|
21
|
+
bundlePath = excluded.bundlePath
|
|
22
|
+
`;
|
|
23
|
+
export class Shard {
|
|
24
|
+
db;
|
|
25
|
+
insertStmt;
|
|
26
|
+
byDecisionStmt;
|
|
27
|
+
byAgentRangeStmt;
|
|
28
|
+
byAgentSchemaStmt;
|
|
29
|
+
chainHeadStmt;
|
|
30
|
+
insertMany;
|
|
31
|
+
constructor(path, options = {}) {
|
|
32
|
+
this.db = new Database(path, { readonly: options.readonly ?? false });
|
|
33
|
+
this.db.pragma("journal_mode = WAL");
|
|
34
|
+
this.db.pragma("synchronous = NORMAL");
|
|
35
|
+
if (!options.readonly)
|
|
36
|
+
this.db.exec(DDL);
|
|
37
|
+
this.insertStmt = this.db.prepare(INSERT_SQL);
|
|
38
|
+
// ORDER BY ts, then decisionId as a stable tiebreak within a millisecond.
|
|
39
|
+
this.byDecisionStmt = this.db.prepare("SELECT * FROM attestations WHERE decisionId = ?");
|
|
40
|
+
this.byAgentRangeStmt = this.db.prepare("SELECT * FROM attestations WHERE agentId = ? AND ts >= ? AND ts <= ? ORDER BY ts, decisionId LIMIT ?");
|
|
41
|
+
this.byAgentSchemaStmt = this.db.prepare("SELECT * FROM attestations WHERE agentId = ? AND schemaHash = ? ORDER BY ts, decisionId LIMIT ?");
|
|
42
|
+
this.chainHeadStmt = this.db.prepare("SELECT * FROM attestations WHERE agentId = ? ORDER BY decisionId DESC LIMIT 1");
|
|
43
|
+
this.insertMany = this.db.transaction((rows) => {
|
|
44
|
+
for (const r of rows)
|
|
45
|
+
this.insertStmt.run(r);
|
|
46
|
+
return rows.length;
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
/** Idempotent single-row upsert. */
|
|
50
|
+
insert(row) {
|
|
51
|
+
this.insertStmt.run(row);
|
|
52
|
+
}
|
|
53
|
+
/** Idempotent bulk upsert in one transaction. Throws on any row error. */
|
|
54
|
+
insertBatch(rows) {
|
|
55
|
+
return this.insertMany(rows);
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Idempotent bulk upsert with per-row error isolation, in one transaction.
|
|
59
|
+
* A row that violates a constraint (e.g. a conflicting duplicate decisionId)
|
|
60
|
+
* is skipped and counted in `failed` — the good rows still commit. This is
|
|
61
|
+
* what keeps backfill robust: one bad bundle cannot roll back a whole shard.
|
|
62
|
+
*/
|
|
63
|
+
insertResilient(rows) {
|
|
64
|
+
let written = 0;
|
|
65
|
+
let failed = 0;
|
|
66
|
+
const run = this.db.transaction((rs) => {
|
|
67
|
+
for (const r of rs) {
|
|
68
|
+
try {
|
|
69
|
+
this.insertStmt.run(r);
|
|
70
|
+
written++;
|
|
71
|
+
}
|
|
72
|
+
catch {
|
|
73
|
+
failed++;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
});
|
|
77
|
+
run(rows);
|
|
78
|
+
return { written, failed };
|
|
79
|
+
}
|
|
80
|
+
byDecisionId(decisionId) {
|
|
81
|
+
return this.byDecisionStmt.get(decisionId);
|
|
82
|
+
}
|
|
83
|
+
byAgentRange(agentId, fromTs, toTs, limit = -1) {
|
|
84
|
+
return this.byAgentRangeStmt.all(agentId, fromTs, toTs, limit);
|
|
85
|
+
}
|
|
86
|
+
byAgentSchema(agentId, schemaHash, limit = -1) {
|
|
87
|
+
return this.byAgentSchemaStmt.all(agentId, schemaHash, limit);
|
|
88
|
+
}
|
|
89
|
+
/** The chain tip for an agent: the latest decisionId (ULID is chronological). */
|
|
90
|
+
chainHead(agentId) {
|
|
91
|
+
return this.chainHeadStmt.get(agentId);
|
|
92
|
+
}
|
|
93
|
+
count() {
|
|
94
|
+
return this.db.prepare("SELECT count(*) AS c FROM attestations").get().c;
|
|
95
|
+
}
|
|
96
|
+
close() {
|
|
97
|
+
this.db.close();
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
//# sourceMappingURL=shard.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"shard.js","sourceRoot":"","sources":["../src/shard.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AACtC,OAAO,EAAE,GAAG,EAAiB,MAAM,aAAa,CAAC;AAMjD,MAAM,UAAU,GAAG;;;;;;;;;;;;;;CAclB,CAAC;AAEF,MAAM,OAAO,KAAK;IACP,EAAE,CAAoB;IACd,UAAU,CAAiC;IAC3C,cAAc,CAAqB;IACnC,gBAAgB,CAAqB;IACrC,iBAAiB,CAAqB;IACtC,aAAa,CAAqB;IAClC,UAAU,CAAqD;IAEhF,YAAY,IAAY,EAAE,UAAwB,EAAE;QAClD,IAAI,CAAC,EAAE,GAAG,IAAI,QAAQ,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,KAAK,EAAE,CAAC,CAAC;QACtE,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC;QACrC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,sBAAsB,CAAC,CAAC;QACvC,IAAI,CAAC,OAAO,CAAC,QAAQ;YAAE,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAEzC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QAC9C,0EAA0E;QAC1E,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CACnC,iDAAiD,CAClD,CAAC;QACF,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CACrC,sGAAsG,CACvG,CAAC;QACF,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CACtC,iGAAiG,CAClG,CAAC;QACF,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAClC,+EAA+E,CAChF,CAAC;QACF,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC,IAAgB,EAAE,EAAE;YACzD,KAAK,MAAM,CAAC,IAAI,IAAI;gBAAE,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAC7C,OAAO,IAAI,CAAC,MAAM,CAAC;QACrB,CAAC,CAAC,CAAC;IACL,CAAC;IAED,oCAAoC;IACpC,MAAM,CAAC,GAAa;QAClB,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC3B,CAAC;IAED,0EAA0E;IAC1E,WAAW,CAAC,IAAgB;QAC1B,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;IAED;;;;;OAKG;IACH,eAAe,CAAC,IAAgB;QAC9B,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC,EAAc,EAAE,EAAE;YACjD,KAAK,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC;gBACnB,IAAI,CAAC;oBACH,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;oBACvB,OAAO,EAAE,CAAC;gBACZ,CAAC;gBAAC,MAAM,CAAC;oBACP,MAAM,EAAE,CAAC;gBACX,CAAC;YACH,CAAC;QACH,CAAC,CAAC,CAAC;QACH,GAAG,CAAC,IAAI,CAAC,CAAC;QACV,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;IAC7B,CAAC;IAED,YAAY,CAAC,UAAkB;QAC7B,OAAO,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,UAAU,CAAyB,CAAC;IACrE,CAAC;IAED,YAAY,CAAC,OAAe,EAAE,MAAc,EAAE,IAAY,EAAE,KAAK,GAAG,CAAC,CAAC;QACpE,OAAO,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,CAAe,CAAC;IAC/E,CAAC;IAED,aAAa,CAAC,OAAe,EAAE,UAAkB,EAAE,KAAK,GAAG,CAAC,CAAC;QAC3D,OAAO,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,OAAO,EAAE,UAAU,EAAE,KAAK,CAAe,CAAC;IAC9E,CAAC;IAED,iFAAiF;IACjF,SAAS,CAAC,OAAe;QACvB,OAAO,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,OAAO,CAAyB,CAAC;IACjE,CAAC;IAED,KAAK;QACH,OAAQ,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,wCAAwC,CAAC,CAAC,GAAG,EAAoB,CAAC,CAAC,CAAC;IAC9F,CAAC;IAED,KAAK;QACH,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;IAClB,CAAC;CACF"}
|
package/dist/store.d.ts
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { type IndexRow } from "./schema.js";
|
|
2
|
+
export interface IndexOptions {
|
|
3
|
+
readonly?: boolean;
|
|
4
|
+
}
|
|
5
|
+
export declare class Index {
|
|
6
|
+
private readonly dir;
|
|
7
|
+
private readonly readOnly;
|
|
8
|
+
private readonly shards;
|
|
9
|
+
private dayCache;
|
|
10
|
+
constructor(dir: string, options?: IndexOptions);
|
|
11
|
+
/** Day keys with an existing shard file, ascending. Cached per instance. */
|
|
12
|
+
shardDays(): string[];
|
|
13
|
+
private shardForDay;
|
|
14
|
+
/** Idempotent write, routed to the row's day shard. */
|
|
15
|
+
write(row: IndexRow): void;
|
|
16
|
+
/** Idempotent bulk write. Rows may span days; each is routed to its shard. */
|
|
17
|
+
writeBatch(rows: IndexRow[]): number;
|
|
18
|
+
/**
|
|
19
|
+
* Idempotent bulk write with per-row error isolation (for backfill). A row
|
|
20
|
+
* that fails to insert (e.g. a conflicting duplicate decisionId) is counted
|
|
21
|
+
* in `failed` rather than aborting its shard. Rows must already have a valid
|
|
22
|
+
* day key (callers validate `ts` up front).
|
|
23
|
+
*/
|
|
24
|
+
writeBatchResilient(rows: IndexRow[]): {
|
|
25
|
+
written: number;
|
|
26
|
+
failed: number;
|
|
27
|
+
};
|
|
28
|
+
private groupByDay;
|
|
29
|
+
private openShards;
|
|
30
|
+
byDecisionId(decisionId: string): IndexRow | undefined;
|
|
31
|
+
byAgentRange(agentId: string, fromTs: string, toTs: string, limit?: number): IndexRow[];
|
|
32
|
+
byAgentSchema(agentId: string, schemaHash: string, limit?: number): IndexRow[];
|
|
33
|
+
/** Global chain tip for an agent across all shards (max decisionId). */
|
|
34
|
+
chainHead(agentId: string): IndexRow | undefined;
|
|
35
|
+
count(): number;
|
|
36
|
+
close(): void;
|
|
37
|
+
}
|
|
38
|
+
//# sourceMappingURL=store.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../src/store.ts"],"names":[],"mappings":"AAQA,OAAO,EAIL,KAAK,QAAQ,EACd,MAAM,aAAa,CAAC;AAErB,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;AAED,qBAAa,KAAK;IAChB,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAS;IAC7B,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAU;IACnC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA4B;IACnD,OAAO,CAAC,QAAQ,CAAyB;gBAE7B,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,YAAiB;IAMnD,4EAA4E;IAC5E,SAAS,IAAI,MAAM,EAAE;IAgBrB,OAAO,CAAC,WAAW;IAenB,uDAAuD;IACvD,KAAK,CAAC,GAAG,EAAE,QAAQ,GAAG,IAAI;IAM1B,8EAA8E;IAC9E,UAAU,CAAC,IAAI,EAAE,QAAQ,EAAE,GAAG,MAAM;IAQpC;;;;;OAKG;IACH,mBAAmB,CAAC,IAAI,EAAE,QAAQ,EAAE,GAAG;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE;IAW1E,OAAO,CAAC,UAAU;IASlB,OAAO,CAAC,UAAU;IAMlB,YAAY,CAAC,UAAU,EAAE,MAAM,GAAG,QAAQ,GAAG,SAAS;IAQtD,YAAY,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,SAAK,GAAG,QAAQ,EAAE;IAanF,aAAa,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,KAAK,SAAK,GAAG,QAAQ,EAAE;IAS1E,wEAAwE;IACxE,SAAS,CAAC,OAAO,EAAE,MAAM,GAAG,QAAQ,GAAG,SAAS;IAShD,KAAK,IAAI,MAAM;IAIf,KAAK,IAAI,IAAI;CAId"}
|
package/dist/store.js
ADDED
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Index — a directory of day-shards (tasks/P2.md). Routes writes to the shard
|
|
3
|
+
* for a row's `ts` day, and fans the four queries out across shards, merging.
|
|
4
|
+
* Shard handles are opened lazily and cached.
|
|
5
|
+
*/
|
|
6
|
+
import { mkdirSync, readdirSync } from "node:fs";
|
|
7
|
+
import { join } from "node:path";
|
|
8
|
+
import { Shard } from "./shard.js";
|
|
9
|
+
import { dayKeyFromFileName, shardFileName, shardKeyForTs, } from "./schema.js";
|
|
10
|
+
export class Index {
|
|
11
|
+
dir;
|
|
12
|
+
readOnly;
|
|
13
|
+
shards = new Map();
|
|
14
|
+
dayCache = null;
|
|
15
|
+
constructor(dir, options = {}) {
|
|
16
|
+
this.dir = dir;
|
|
17
|
+
this.readOnly = options.readonly ?? false;
|
|
18
|
+
if (!this.readOnly)
|
|
19
|
+
mkdirSync(dir, { recursive: true });
|
|
20
|
+
}
|
|
21
|
+
/** Day keys with an existing shard file, ascending. Cached per instance. */
|
|
22
|
+
shardDays() {
|
|
23
|
+
if (this.dayCache)
|
|
24
|
+
return this.dayCache;
|
|
25
|
+
let names;
|
|
26
|
+
try {
|
|
27
|
+
names = readdirSync(this.dir);
|
|
28
|
+
}
|
|
29
|
+
catch {
|
|
30
|
+
this.dayCache = [];
|
|
31
|
+
return this.dayCache;
|
|
32
|
+
}
|
|
33
|
+
this.dayCache = names
|
|
34
|
+
.map(dayKeyFromFileName)
|
|
35
|
+
.filter((k) => k !== null)
|
|
36
|
+
.sort();
|
|
37
|
+
return this.dayCache;
|
|
38
|
+
}
|
|
39
|
+
shardForDay(day, createIfMissing) {
|
|
40
|
+
const cached = this.shards.get(day);
|
|
41
|
+
if (cached)
|
|
42
|
+
return cached;
|
|
43
|
+
const exists = this.shardDays().includes(day);
|
|
44
|
+
// Readonly / query paths must not create a shard file that isn't there.
|
|
45
|
+
if (!exists && (this.readOnly || !createIfMissing))
|
|
46
|
+
return undefined;
|
|
47
|
+
const shard = new Shard(join(this.dir, shardFileName(day)), { readonly: this.readOnly });
|
|
48
|
+
this.shards.set(day, shard);
|
|
49
|
+
if (!exists && this.dayCache) {
|
|
50
|
+
this.dayCache.push(day);
|
|
51
|
+
this.dayCache.sort();
|
|
52
|
+
}
|
|
53
|
+
return shard;
|
|
54
|
+
}
|
|
55
|
+
/** Idempotent write, routed to the row's day shard. */
|
|
56
|
+
write(row) {
|
|
57
|
+
const day = shardKeyForTs(row.ts);
|
|
58
|
+
const shard = this.shardForDay(day, true);
|
|
59
|
+
shard.insert(row);
|
|
60
|
+
}
|
|
61
|
+
/** Idempotent bulk write. Rows may span days; each is routed to its shard. */
|
|
62
|
+
writeBatch(rows) {
|
|
63
|
+
let n = 0;
|
|
64
|
+
for (const [day, dayRows] of this.groupByDay(rows)) {
|
|
65
|
+
n += this.shardForDay(day, true).insertBatch(dayRows);
|
|
66
|
+
}
|
|
67
|
+
return n;
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Idempotent bulk write with per-row error isolation (for backfill). A row
|
|
71
|
+
* that fails to insert (e.g. a conflicting duplicate decisionId) is counted
|
|
72
|
+
* in `failed` rather than aborting its shard. Rows must already have a valid
|
|
73
|
+
* day key (callers validate `ts` up front).
|
|
74
|
+
*/
|
|
75
|
+
writeBatchResilient(rows) {
|
|
76
|
+
let written = 0;
|
|
77
|
+
let failed = 0;
|
|
78
|
+
for (const [day, dayRows] of this.groupByDay(rows)) {
|
|
79
|
+
const r = this.shardForDay(day, true).insertResilient(dayRows);
|
|
80
|
+
written += r.written;
|
|
81
|
+
failed += r.failed;
|
|
82
|
+
}
|
|
83
|
+
return { written, failed };
|
|
84
|
+
}
|
|
85
|
+
groupByDay(rows) {
|
|
86
|
+
const byDay = new Map();
|
|
87
|
+
for (const r of rows) {
|
|
88
|
+
const day = shardKeyForTs(r.ts);
|
|
89
|
+
(byDay.get(day) ?? byDay.set(day, []).get(day)).push(r);
|
|
90
|
+
}
|
|
91
|
+
return byDay;
|
|
92
|
+
}
|
|
93
|
+
openShards() {
|
|
94
|
+
return this.shardDays()
|
|
95
|
+
.map((day) => this.shardForDay(day, false))
|
|
96
|
+
.filter((s) => s !== undefined);
|
|
97
|
+
}
|
|
98
|
+
byDecisionId(decisionId) {
|
|
99
|
+
for (const shard of this.openShards()) {
|
|
100
|
+
const row = shard.byDecisionId(decisionId);
|
|
101
|
+
if (row)
|
|
102
|
+
return row;
|
|
103
|
+
}
|
|
104
|
+
return undefined;
|
|
105
|
+
}
|
|
106
|
+
byAgentRange(agentId, fromTs, toTs, limit = -1) {
|
|
107
|
+
const fromDay = shardKeyForTs(fromTs);
|
|
108
|
+
const toDay = shardKeyForTs(toTs);
|
|
109
|
+
const rows = [];
|
|
110
|
+
for (const day of this.shardDays()) {
|
|
111
|
+
if (day < fromDay || day > toDay)
|
|
112
|
+
continue;
|
|
113
|
+
const shard = this.shardForDay(day, false);
|
|
114
|
+
if (shard)
|
|
115
|
+
rows.push(...shard.byAgentRange(agentId, fromTs, toTs, limit));
|
|
116
|
+
}
|
|
117
|
+
rows.sort((a, b) => (a.ts < b.ts ? -1 : a.ts > b.ts ? 1 : a.decisionId < b.decisionId ? -1 : 1));
|
|
118
|
+
return limit >= 0 ? rows.slice(0, limit) : rows;
|
|
119
|
+
}
|
|
120
|
+
byAgentSchema(agentId, schemaHash, limit = -1) {
|
|
121
|
+
const rows = [];
|
|
122
|
+
for (const shard of this.openShards()) {
|
|
123
|
+
rows.push(...shard.byAgentSchema(agentId, schemaHash, limit));
|
|
124
|
+
}
|
|
125
|
+
rows.sort((a, b) => (a.ts < b.ts ? -1 : a.ts > b.ts ? 1 : a.decisionId < b.decisionId ? -1 : 1));
|
|
126
|
+
return limit >= 0 ? rows.slice(0, limit) : rows;
|
|
127
|
+
}
|
|
128
|
+
/** Global chain tip for an agent across all shards (max decisionId). */
|
|
129
|
+
chainHead(agentId) {
|
|
130
|
+
let head;
|
|
131
|
+
for (const shard of this.openShards()) {
|
|
132
|
+
const h = shard.chainHead(agentId);
|
|
133
|
+
if (h && (!head || h.decisionId > head.decisionId))
|
|
134
|
+
head = h;
|
|
135
|
+
}
|
|
136
|
+
return head;
|
|
137
|
+
}
|
|
138
|
+
count() {
|
|
139
|
+
return this.openShards().reduce((n, s) => n + s.count(), 0);
|
|
140
|
+
}
|
|
141
|
+
close() {
|
|
142
|
+
for (const shard of this.shards.values())
|
|
143
|
+
shard.close();
|
|
144
|
+
this.shards.clear();
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
//# sourceMappingURL=store.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"store.js","sourceRoot":"","sources":["../src/store.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AACjD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,EACL,kBAAkB,EAClB,aAAa,EACb,aAAa,GAEd,MAAM,aAAa,CAAC;AAMrB,MAAM,OAAO,KAAK;IACC,GAAG,CAAS;IACZ,QAAQ,CAAU;IAClB,MAAM,GAAG,IAAI,GAAG,EAAiB,CAAC;IAC3C,QAAQ,GAAoB,IAAI,CAAC;IAEzC,YAAY,GAAW,EAAE,UAAwB,EAAE;QACjD,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;QACf,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,KAAK,CAAC;QAC1C,IAAI,CAAC,IAAI,CAAC,QAAQ;YAAE,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC1D,CAAC;IAED,4EAA4E;IAC5E,SAAS;QACP,IAAI,IAAI,CAAC,QAAQ;YAAE,OAAO,IAAI,CAAC,QAAQ,CAAC;QACxC,IAAI,KAAe,CAAC;QACpB,IAAI,CAAC;YACH,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAChC,CAAC;QAAC,MAAM,CAAC;YACP,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC;YACnB,OAAO,IAAI,CAAC,QAAQ,CAAC;QACvB,CAAC;QACD,IAAI,CAAC,QAAQ,GAAG,KAAK;aAClB,GAAG,CAAC,kBAAkB,CAAC;aACvB,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC;aACtC,IAAI,EAAE,CAAC;QACV,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAEO,WAAW,CAAC,GAAW,EAAE,eAAwB;QACvD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACpC,IAAI,MAAM;YAAE,OAAO,MAAM,CAAC;QAC1B,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;QAC9C,wEAAwE;QACxE,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,CAAC,eAAe,CAAC;YAAE,OAAO,SAAS,CAAC;QACrE,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,aAAa,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;QACzF,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC5B,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC7B,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACxB,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QACvB,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,uDAAuD;IACvD,KAAK,CAAC,GAAa;QACjB,MAAM,GAAG,GAAG,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAClC,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,IAAI,CAAE,CAAC;QAC3C,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IACpB,CAAC;IAED,8EAA8E;IAC9E,UAAU,CAAC,IAAgB;QACzB,IAAI,CAAC,GAAG,CAAC,CAAC;QACV,KAAK,MAAM,CAAC,GAAG,EAAE,OAAO,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YACnD,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,IAAI,CAAE,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;QACzD,CAAC;QACD,OAAO,CAAC,CAAC;IACX,CAAC;IAED;;;;;OAKG;IACH,mBAAmB,CAAC,IAAgB;QAClC,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,KAAK,MAAM,CAAC,GAAG,EAAE,OAAO,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YACnD,MAAM,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,IAAI,CAAE,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;YAChE,OAAO,IAAI,CAAC,CAAC,OAAO,CAAC;YACrB,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC;QACrB,CAAC;QACD,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;IAC7B,CAAC;IAEO,UAAU,CAAC,IAAgB;QACjC,MAAM,KAAK,GAAG,IAAI,GAAG,EAAsB,CAAC;QAC5C,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;YACrB,MAAM,GAAG,GAAG,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAChC,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC3D,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAEO,UAAU;QAChB,OAAO,IAAI,CAAC,SAAS,EAAE;aACpB,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;aAC1C,MAAM,CAAC,CAAC,CAAC,EAAc,EAAE,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC;IAChD,CAAC;IAED,YAAY,CAAC,UAAkB;QAC7B,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,UAAU,EAAE,EAAE,CAAC;YACtC,MAAM,GAAG,GAAG,KAAK,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC;YAC3C,IAAI,GAAG;gBAAE,OAAO,GAAG,CAAC;QACtB,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,YAAY,CAAC,OAAe,EAAE,MAAc,EAAE,IAAY,EAAE,KAAK,GAAG,CAAC,CAAC;QACpE,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;QACtC,MAAM,KAAK,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;QAClC,MAAM,IAAI,GAAe,EAAE,CAAC;QAC5B,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,SAAS,EAAE,EAAE,CAAC;YACnC,IAAI,GAAG,GAAG,OAAO,IAAI,GAAG,GAAG,KAAK;gBAAE,SAAS;YAC3C,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YAC3C,IAAI,KAAK;gBAAE,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,YAAY,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;QAC5E,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACjG,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAClD,CAAC;IAED,aAAa,CAAC,OAAe,EAAE,UAAkB,EAAE,KAAK,GAAG,CAAC,CAAC;QAC3D,MAAM,IAAI,GAAe,EAAE,CAAC;QAC5B,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,UAAU,EAAE,EAAE,CAAC;YACtC,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,aAAa,CAAC,OAAO,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC,CAAC;QAChE,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACjG,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAClD,CAAC;IAED,wEAAwE;IACxE,SAAS,CAAC,OAAe;QACvB,IAAI,IAA0B,CAAC;QAC/B,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,UAAU,EAAE,EAAE,CAAC;YACtC,MAAM,CAAC,GAAG,KAAK,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;YACnC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;gBAAE,IAAI,GAAG,CAAC,CAAC;QAC/D,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,KAAK;QACH,OAAO,IAAI,CAAC,UAAU,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC;IAC9D,CAAC;IAED,KAAK;QACH,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE;YAAE,KAAK,CAAC,KAAK,EAAE,CAAC;QACxD,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;IACtB,CAAC;CACF"}
|
package/package.json
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@rubric-protocol/attest-index",
|
|
3
|
+
"version": "1.0.0",
|
|
4
|
+
"type": "module",
|
|
5
|
+
"description": "Rubric attestation index: SQLite day-shard writer, backfill, query lib",
|
|
6
|
+
"license": "Apache-2.0",
|
|
7
|
+
"repository": {
|
|
8
|
+
"type": "git",
|
|
9
|
+
"url": "git+https://github.com/0xsims/rubric-attest.git",
|
|
10
|
+
"directory": "packages/attest-index"
|
|
11
|
+
},
|
|
12
|
+
"main": "./dist/index.js",
|
|
13
|
+
"types": "./dist/index.d.ts",
|
|
14
|
+
"exports": {
|
|
15
|
+
".": {
|
|
16
|
+
"development": "./src/index.ts",
|
|
17
|
+
"types": "./dist/index.d.ts",
|
|
18
|
+
"default": "./dist/index.js"
|
|
19
|
+
}
|
|
20
|
+
},
|
|
21
|
+
"bin": {
|
|
22
|
+
"rubric-index-backfill": "./dist/cli/backfill.js"
|
|
23
|
+
},
|
|
24
|
+
"files": ["dist", "src"],
|
|
25
|
+
"publishConfig": { "access": "public" },
|
|
26
|
+
"scripts": {
|
|
27
|
+
"prepublishOnly": "npm run build",
|
|
28
|
+
"build": "tsc -p tsconfig.build.json",
|
|
29
|
+
"test": "vitest run"
|
|
30
|
+
},
|
|
31
|
+
"dependencies": {
|
|
32
|
+
"@rubric-protocol/attest-decision": "^1.0.0",
|
|
33
|
+
"better-sqlite3": "^11.3.0"
|
|
34
|
+
}
|
|
35
|
+
}
|
package/src/backfill.ts
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Backfill the index from a bundle-store directory (tasks/P2.md).
|
|
3
|
+
*
|
|
4
|
+
* Walks the store for `*.json` bundles and upserts a row per bundle. Idempotent
|
|
5
|
+
* (upsert keyed on attestationId), so re-running is a no-op and the index is
|
|
6
|
+
* fully rebuildable from the bundles alone. `bundlePath` is stored relative to
|
|
7
|
+
* the store root so the index stays portable.
|
|
8
|
+
*/
|
|
9
|
+
import { readdirSync, readFileSync } from "node:fs";
|
|
10
|
+
import { join, relative } from "node:path";
|
|
11
|
+
import { Index } from "./store.js";
|
|
12
|
+
import { rowFromBundle, shardKeyForTs, type AttestationBundle, type IndexRow } from "./schema.js";
|
|
13
|
+
|
|
14
|
+
export interface BackfillResult {
|
|
15
|
+
bundleFiles: number;
|
|
16
|
+
rowsWritten: number;
|
|
17
|
+
/** Bundles that parsed and validated but failed to insert (e.g. a conflicting duplicate decisionId). */
|
|
18
|
+
failed: number;
|
|
19
|
+
/** Files that were not valid bundles (parse error, wrong shape, or bad ts). */
|
|
20
|
+
skipped: number;
|
|
21
|
+
days: string[];
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function walkJson(root: string): string[] {
|
|
25
|
+
const out: string[] = [];
|
|
26
|
+
const stack = [root];
|
|
27
|
+
while (stack.length > 0) {
|
|
28
|
+
const dir = stack.pop()!;
|
|
29
|
+
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
|
30
|
+
const full = join(dir, entry.name);
|
|
31
|
+
if (entry.isDirectory()) stack.push(full);
|
|
32
|
+
else if (entry.isFile() && entry.name.endsWith(".json")) out.push(full);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
out.sort();
|
|
36
|
+
return out;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function isBundle(value: unknown): value is AttestationBundle {
|
|
40
|
+
if (typeof value !== "object" || value === null) return false;
|
|
41
|
+
const b = value as Record<string, unknown>;
|
|
42
|
+
if (typeof b.attestationId !== "string" || b.attestationId.length === 0) return false;
|
|
43
|
+
const d = b.dar as Record<string, unknown> | undefined;
|
|
44
|
+
if (!d || typeof d !== "object") return false;
|
|
45
|
+
// Validate EVERY column the index requires as NOT NULL, so a malformed bundle
|
|
46
|
+
// is skipped up front rather than throwing a NOT NULL error mid-insert.
|
|
47
|
+
const str = (x: unknown) => typeof x === "string";
|
|
48
|
+
return (
|
|
49
|
+
str(d.decisionId) &&
|
|
50
|
+
str(d.agentId) &&
|
|
51
|
+
str(d.ts) &&
|
|
52
|
+
str(d.schemaHash) &&
|
|
53
|
+
str(d.decisionHash) &&
|
|
54
|
+
str(d.leafType) &&
|
|
55
|
+
(d.prev === null || str(d.prev))
|
|
56
|
+
);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/** Backfill (or rebuild) the index at `indexDir` from bundles under `storeDir`. */
|
|
60
|
+
export function backfill(storeDir: string, indexDir: string): BackfillResult {
|
|
61
|
+
const index = new Index(indexDir);
|
|
62
|
+
try {
|
|
63
|
+
const files = walkJson(storeDir);
|
|
64
|
+
const rows: IndexRow[] = [];
|
|
65
|
+
let skipped = 0;
|
|
66
|
+
for (const file of files) {
|
|
67
|
+
let parsed: unknown;
|
|
68
|
+
try {
|
|
69
|
+
parsed = JSON.parse(readFileSync(file, "utf8"));
|
|
70
|
+
} catch {
|
|
71
|
+
skipped++;
|
|
72
|
+
continue;
|
|
73
|
+
}
|
|
74
|
+
if (!isBundle(parsed)) {
|
|
75
|
+
skipped++;
|
|
76
|
+
continue;
|
|
77
|
+
}
|
|
78
|
+
// Reject a ts that has no valid day key here, so grouping can't throw.
|
|
79
|
+
try {
|
|
80
|
+
shardKeyForTs(parsed.dar.ts);
|
|
81
|
+
} catch {
|
|
82
|
+
skipped++;
|
|
83
|
+
continue;
|
|
84
|
+
}
|
|
85
|
+
rows.push(rowFromBundle(parsed, relative(storeDir, file)));
|
|
86
|
+
}
|
|
87
|
+
const { written, failed } = index.writeBatchResilient(rows);
|
|
88
|
+
return {
|
|
89
|
+
bundleFiles: files.length,
|
|
90
|
+
rowsWritten: written,
|
|
91
|
+
failed,
|
|
92
|
+
skipped,
|
|
93
|
+
days: index.shardDays(),
|
|
94
|
+
};
|
|
95
|
+
} finally {
|
|
96
|
+
index.close();
|
|
97
|
+
}
|
|
98
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* CLI: rebuild the attestation index from a bundle store.
|
|
4
|
+
* rubric-index-backfill <bundle-store-dir> <index-dir>
|
|
5
|
+
*/
|
|
6
|
+
import { backfill } from "../backfill.js";
|
|
7
|
+
|
|
8
|
+
function main(argv: string[]): void {
|
|
9
|
+
const [storeDir, indexDir] = argv;
|
|
10
|
+
if (!storeDir || !indexDir) {
|
|
11
|
+
process.stderr.write("usage: rubric-index-backfill <bundle-store-dir> <index-dir>\n");
|
|
12
|
+
process.exit(2);
|
|
13
|
+
}
|
|
14
|
+
const result = backfill(storeDir, indexDir);
|
|
15
|
+
process.stdout.write(JSON.stringify(result, null, 2) + "\n");
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
main(process.argv.slice(2));
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @rubric-protocol/attest-index — SQLite day-shard attestation index (tasks/P2.md).
|
|
3
|
+
*
|
|
4
|
+
* Writer + query lib over per-day WAL shards, plus a backfill that rebuilds the
|
|
5
|
+
* index from a bundle store. The bundle store is the source of truth; the index
|
|
6
|
+
* is a rebuildable cache.
|
|
7
|
+
*/
|
|
8
|
+
export {
|
|
9
|
+
type IndexRow,
|
|
10
|
+
type AttestationBundle,
|
|
11
|
+
rowFromBundle,
|
|
12
|
+
shardKeyForTs,
|
|
13
|
+
shardFileName,
|
|
14
|
+
dayKeyFromFileName,
|
|
15
|
+
DDL,
|
|
16
|
+
} from "./schema.js";
|
|
17
|
+
export { Shard, type ShardOptions } from "./shard.js";
|
|
18
|
+
export { Index, type IndexOptions } from "./store.js";
|
|
19
|
+
export { backfill, type BackfillResult } from "./backfill.js";
|
package/src/schema.ts
ADDED
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Attestation index schema (tasks/P2.md). One row per attested leaf; the row
|
|
3
|
+
* mirrors the DAR core identity plus the service `attestationId` and a pointer
|
|
4
|
+
* back to the bundle it was derived from (so the index is a cache, and the
|
|
5
|
+
* bundle store remains the source of truth — the index is rebuildable).
|
|
6
|
+
*/
|
|
7
|
+
import type { DarCore } from "@rubric-protocol/attest-decision";
|
|
8
|
+
|
|
9
|
+
export interface IndexRow {
|
|
10
|
+
attestationId: string;
|
|
11
|
+
decisionId: string;
|
|
12
|
+
agentId: string;
|
|
13
|
+
schemaHash: string;
|
|
14
|
+
decisionHash: string;
|
|
15
|
+
prev: string | null;
|
|
16
|
+
ts: string;
|
|
17
|
+
leafType: string;
|
|
18
|
+
bundlePath: string;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/** A stored attestation bundle: the DAR core plus its service `attestationId`. */
|
|
22
|
+
export interface AttestationBundle {
|
|
23
|
+
attestationId: string;
|
|
24
|
+
dar: DarCore;
|
|
25
|
+
// Envelope extras (merkleProof, anchorRef, signature, extensions) may also be
|
|
26
|
+
// present in a bundle; they are not indexed here (see spec/dar-0.1.md §6).
|
|
27
|
+
[k: string]: unknown;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** Derive an index row from a bundle and the path it was read from. */
|
|
31
|
+
export function rowFromBundle(bundle: AttestationBundle, bundlePath: string): IndexRow {
|
|
32
|
+
const d = bundle.dar;
|
|
33
|
+
return {
|
|
34
|
+
attestationId: bundle.attestationId,
|
|
35
|
+
decisionId: d.decisionId,
|
|
36
|
+
agentId: d.agentId,
|
|
37
|
+
schemaHash: d.schemaHash,
|
|
38
|
+
decisionHash: d.decisionHash,
|
|
39
|
+
prev: d.prev,
|
|
40
|
+
ts: d.ts,
|
|
41
|
+
leafType: d.leafType,
|
|
42
|
+
bundlePath,
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** UTC day key (YYYY-MM-DD) that a row's `ts` shards into. */
|
|
47
|
+
export function shardKeyForTs(ts: string): string {
|
|
48
|
+
const key = ts.slice(0, 10);
|
|
49
|
+
if (!/^\d{4}-\d{2}-\d{2}$/.test(key)) {
|
|
50
|
+
throw new Error(`index: cannot derive shard day from ts '${ts}'`);
|
|
51
|
+
}
|
|
52
|
+
return key;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export const SHARD_FILE_PREFIX = "attest-";
|
|
56
|
+
export const SHARD_FILE_SUFFIX = ".sqlite";
|
|
57
|
+
|
|
58
|
+
export function shardFileName(dayKey: string): string {
|
|
59
|
+
return `${SHARD_FILE_PREFIX}${dayKey}${SHARD_FILE_SUFFIX}`;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/** Extract the day key from a shard file name, or null if it is not a shard. */
|
|
63
|
+
export function dayKeyFromFileName(name: string): string | null {
|
|
64
|
+
if (!name.startsWith(SHARD_FILE_PREFIX) || !name.endsWith(SHARD_FILE_SUFFIX)) return null;
|
|
65
|
+
const key = name.slice(SHARD_FILE_PREFIX.length, name.length - SHARD_FILE_SUFFIX.length);
|
|
66
|
+
return /^\d{4}-\d{2}-\d{2}$/.test(key) ? key : null;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/** DDL. Indexes back the four queries in tasks/P2.md. */
|
|
70
|
+
export const DDL = `
|
|
71
|
+
CREATE TABLE IF NOT EXISTS attestations (
|
|
72
|
+
attestationId TEXT PRIMARY KEY,
|
|
73
|
+
decisionId TEXT NOT NULL,
|
|
74
|
+
agentId TEXT NOT NULL,
|
|
75
|
+
schemaHash TEXT NOT NULL,
|
|
76
|
+
decisionHash TEXT NOT NULL,
|
|
77
|
+
prev TEXT,
|
|
78
|
+
ts TEXT NOT NULL,
|
|
79
|
+
leafType TEXT NOT NULL,
|
|
80
|
+
bundlePath TEXT NOT NULL
|
|
81
|
+
);
|
|
82
|
+
CREATE UNIQUE INDEX IF NOT EXISTS ux_decisionId ON attestations(decisionId);
|
|
83
|
+
CREATE INDEX IF NOT EXISTS ix_agent_ts ON attestations(agentId, ts, decisionId);
|
|
84
|
+
CREATE INDEX IF NOT EXISTS ix_agent_schema ON attestations(agentId, schemaHash, ts, decisionId);
|
|
85
|
+
CREATE INDEX IF NOT EXISTS ix_agent_decisionId ON attestations(agentId, decisionId);
|
|
86
|
+
CREATE INDEX IF NOT EXISTS ix_agent_prev ON attestations(agentId, prev);
|
|
87
|
+
`;
|
package/src/shard.ts
ADDED
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A single day-shard: one WAL-mode SQLite file with the four indexed queries
|
|
3
|
+
* from tasks/P2.md. `insert` is idempotent (keyed on attestationId), which is
|
|
4
|
+
* what makes backfill idempotent and the index rebuildable from bundles alone.
|
|
5
|
+
*/
|
|
6
|
+
import Database from "better-sqlite3";
|
|
7
|
+
import { DDL, type IndexRow } from "./schema.js";
|
|
8
|
+
|
|
9
|
+
export interface ShardOptions {
|
|
10
|
+
readonly?: boolean;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
const INSERT_SQL = `
|
|
14
|
+
INSERT INTO attestations
|
|
15
|
+
(attestationId, decisionId, agentId, schemaHash, decisionHash, prev, ts, leafType, bundlePath)
|
|
16
|
+
VALUES
|
|
17
|
+
(@attestationId, @decisionId, @agentId, @schemaHash, @decisionHash, @prev, @ts, @leafType, @bundlePath)
|
|
18
|
+
ON CONFLICT(attestationId) DO UPDATE SET
|
|
19
|
+
decisionId = excluded.decisionId,
|
|
20
|
+
agentId = excluded.agentId,
|
|
21
|
+
schemaHash = excluded.schemaHash,
|
|
22
|
+
decisionHash = excluded.decisionHash,
|
|
23
|
+
prev = excluded.prev,
|
|
24
|
+
ts = excluded.ts,
|
|
25
|
+
leafType = excluded.leafType,
|
|
26
|
+
bundlePath = excluded.bundlePath
|
|
27
|
+
`;
|
|
28
|
+
|
|
29
|
+
export class Shard {
|
|
30
|
+
readonly db: Database.Database;
|
|
31
|
+
private readonly insertStmt: Database.Statement<[IndexRow]>;
|
|
32
|
+
private readonly byDecisionStmt: Database.Statement;
|
|
33
|
+
private readonly byAgentRangeStmt: Database.Statement;
|
|
34
|
+
private readonly byAgentSchemaStmt: Database.Statement;
|
|
35
|
+
private readonly chainHeadStmt: Database.Statement;
|
|
36
|
+
private readonly insertMany: Database.Transaction<(rows: IndexRow[]) => number>;
|
|
37
|
+
|
|
38
|
+
constructor(path: string, options: ShardOptions = {}) {
|
|
39
|
+
this.db = new Database(path, { readonly: options.readonly ?? false });
|
|
40
|
+
this.db.pragma("journal_mode = WAL");
|
|
41
|
+
this.db.pragma("synchronous = NORMAL");
|
|
42
|
+
if (!options.readonly) this.db.exec(DDL);
|
|
43
|
+
|
|
44
|
+
this.insertStmt = this.db.prepare(INSERT_SQL);
|
|
45
|
+
// ORDER BY ts, then decisionId as a stable tiebreak within a millisecond.
|
|
46
|
+
this.byDecisionStmt = this.db.prepare(
|
|
47
|
+
"SELECT * FROM attestations WHERE decisionId = ?",
|
|
48
|
+
);
|
|
49
|
+
this.byAgentRangeStmt = this.db.prepare(
|
|
50
|
+
"SELECT * FROM attestations WHERE agentId = ? AND ts >= ? AND ts <= ? ORDER BY ts, decisionId LIMIT ?",
|
|
51
|
+
);
|
|
52
|
+
this.byAgentSchemaStmt = this.db.prepare(
|
|
53
|
+
"SELECT * FROM attestations WHERE agentId = ? AND schemaHash = ? ORDER BY ts, decisionId LIMIT ?",
|
|
54
|
+
);
|
|
55
|
+
this.chainHeadStmt = this.db.prepare(
|
|
56
|
+
"SELECT * FROM attestations WHERE agentId = ? ORDER BY decisionId DESC LIMIT 1",
|
|
57
|
+
);
|
|
58
|
+
this.insertMany = this.db.transaction((rows: IndexRow[]) => {
|
|
59
|
+
for (const r of rows) this.insertStmt.run(r);
|
|
60
|
+
return rows.length;
|
|
61
|
+
});
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** Idempotent single-row upsert. */
|
|
65
|
+
insert(row: IndexRow): void {
|
|
66
|
+
this.insertStmt.run(row);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/** Idempotent bulk upsert in one transaction. Throws on any row error. */
|
|
70
|
+
insertBatch(rows: IndexRow[]): number {
|
|
71
|
+
return this.insertMany(rows);
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* Idempotent bulk upsert with per-row error isolation, in one transaction.
|
|
76
|
+
* A row that violates a constraint (e.g. a conflicting duplicate decisionId)
|
|
77
|
+
* is skipped and counted in `failed` — the good rows still commit. This is
|
|
78
|
+
* what keeps backfill robust: one bad bundle cannot roll back a whole shard.
|
|
79
|
+
*/
|
|
80
|
+
insertResilient(rows: IndexRow[]): { written: number; failed: number } {
|
|
81
|
+
let written = 0;
|
|
82
|
+
let failed = 0;
|
|
83
|
+
const run = this.db.transaction((rs: IndexRow[]) => {
|
|
84
|
+
for (const r of rs) {
|
|
85
|
+
try {
|
|
86
|
+
this.insertStmt.run(r);
|
|
87
|
+
written++;
|
|
88
|
+
} catch {
|
|
89
|
+
failed++;
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
});
|
|
93
|
+
run(rows);
|
|
94
|
+
return { written, failed };
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
byDecisionId(decisionId: string): IndexRow | undefined {
|
|
98
|
+
return this.byDecisionStmt.get(decisionId) as IndexRow | undefined;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
byAgentRange(agentId: string, fromTs: string, toTs: string, limit = -1): IndexRow[] {
|
|
102
|
+
return this.byAgentRangeStmt.all(agentId, fromTs, toTs, limit) as IndexRow[];
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
byAgentSchema(agentId: string, schemaHash: string, limit = -1): IndexRow[] {
|
|
106
|
+
return this.byAgentSchemaStmt.all(agentId, schemaHash, limit) as IndexRow[];
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/** The chain tip for an agent: the latest decisionId (ULID is chronological). */
|
|
110
|
+
chainHead(agentId: string): IndexRow | undefined {
|
|
111
|
+
return this.chainHeadStmt.get(agentId) as IndexRow | undefined;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
count(): number {
|
|
115
|
+
return (this.db.prepare("SELECT count(*) AS c FROM attestations").get() as { c: number }).c;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
close(): void {
|
|
119
|
+
this.db.close();
|
|
120
|
+
}
|
|
121
|
+
}
|
package/src/store.ts
ADDED
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Index — a directory of day-shards (tasks/P2.md). Routes writes to the shard
|
|
3
|
+
* for a row's `ts` day, and fans the four queries out across shards, merging.
|
|
4
|
+
* Shard handles are opened lazily and cached.
|
|
5
|
+
*/
|
|
6
|
+
import { mkdirSync, readdirSync } from "node:fs";
|
|
7
|
+
import { join } from "node:path";
|
|
8
|
+
import { Shard } from "./shard.js";
|
|
9
|
+
import {
|
|
10
|
+
dayKeyFromFileName,
|
|
11
|
+
shardFileName,
|
|
12
|
+
shardKeyForTs,
|
|
13
|
+
type IndexRow,
|
|
14
|
+
} from "./schema.js";
|
|
15
|
+
|
|
16
|
+
export interface IndexOptions {
|
|
17
|
+
readonly?: boolean;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export class Index {
|
|
21
|
+
private readonly dir: string;
|
|
22
|
+
private readonly readOnly: boolean;
|
|
23
|
+
private readonly shards = new Map<string, Shard>();
|
|
24
|
+
private dayCache: string[] | null = null;
|
|
25
|
+
|
|
26
|
+
constructor(dir: string, options: IndexOptions = {}) {
|
|
27
|
+
this.dir = dir;
|
|
28
|
+
this.readOnly = options.readonly ?? false;
|
|
29
|
+
if (!this.readOnly) mkdirSync(dir, { recursive: true });
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** Day keys with an existing shard file, ascending. Cached per instance. */
|
|
33
|
+
shardDays(): string[] {
|
|
34
|
+
if (this.dayCache) return this.dayCache;
|
|
35
|
+
let names: string[];
|
|
36
|
+
try {
|
|
37
|
+
names = readdirSync(this.dir);
|
|
38
|
+
} catch {
|
|
39
|
+
this.dayCache = [];
|
|
40
|
+
return this.dayCache;
|
|
41
|
+
}
|
|
42
|
+
this.dayCache = names
|
|
43
|
+
.map(dayKeyFromFileName)
|
|
44
|
+
.filter((k): k is string => k !== null)
|
|
45
|
+
.sort();
|
|
46
|
+
return this.dayCache;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
private shardForDay(day: string, createIfMissing: boolean): Shard | undefined {
|
|
50
|
+
const cached = this.shards.get(day);
|
|
51
|
+
if (cached) return cached;
|
|
52
|
+
const exists = this.shardDays().includes(day);
|
|
53
|
+
// Readonly / query paths must not create a shard file that isn't there.
|
|
54
|
+
if (!exists && (this.readOnly || !createIfMissing)) return undefined;
|
|
55
|
+
const shard = new Shard(join(this.dir, shardFileName(day)), { readonly: this.readOnly });
|
|
56
|
+
this.shards.set(day, shard);
|
|
57
|
+
if (!exists && this.dayCache) {
|
|
58
|
+
this.dayCache.push(day);
|
|
59
|
+
this.dayCache.sort();
|
|
60
|
+
}
|
|
61
|
+
return shard;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** Idempotent write, routed to the row's day shard. */
|
|
65
|
+
write(row: IndexRow): void {
|
|
66
|
+
const day = shardKeyForTs(row.ts);
|
|
67
|
+
const shard = this.shardForDay(day, true)!;
|
|
68
|
+
shard.insert(row);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/** Idempotent bulk write. Rows may span days; each is routed to its shard. */
|
|
72
|
+
writeBatch(rows: IndexRow[]): number {
|
|
73
|
+
let n = 0;
|
|
74
|
+
for (const [day, dayRows] of this.groupByDay(rows)) {
|
|
75
|
+
n += this.shardForDay(day, true)!.insertBatch(dayRows);
|
|
76
|
+
}
|
|
77
|
+
return n;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* Idempotent bulk write with per-row error isolation (for backfill). A row
|
|
82
|
+
* that fails to insert (e.g. a conflicting duplicate decisionId) is counted
|
|
83
|
+
* in `failed` rather than aborting its shard. Rows must already have a valid
|
|
84
|
+
* day key (callers validate `ts` up front).
|
|
85
|
+
*/
|
|
86
|
+
writeBatchResilient(rows: IndexRow[]): { written: number; failed: number } {
|
|
87
|
+
let written = 0;
|
|
88
|
+
let failed = 0;
|
|
89
|
+
for (const [day, dayRows] of this.groupByDay(rows)) {
|
|
90
|
+
const r = this.shardForDay(day, true)!.insertResilient(dayRows);
|
|
91
|
+
written += r.written;
|
|
92
|
+
failed += r.failed;
|
|
93
|
+
}
|
|
94
|
+
return { written, failed };
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
private groupByDay(rows: IndexRow[]): Map<string, IndexRow[]> {
|
|
98
|
+
const byDay = new Map<string, IndexRow[]>();
|
|
99
|
+
for (const r of rows) {
|
|
100
|
+
const day = shardKeyForTs(r.ts);
|
|
101
|
+
(byDay.get(day) ?? byDay.set(day, []).get(day)!).push(r);
|
|
102
|
+
}
|
|
103
|
+
return byDay;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
private openShards(): Shard[] {
|
|
107
|
+
return this.shardDays()
|
|
108
|
+
.map((day) => this.shardForDay(day, false))
|
|
109
|
+
.filter((s): s is Shard => s !== undefined);
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
byDecisionId(decisionId: string): IndexRow | undefined {
|
|
113
|
+
for (const shard of this.openShards()) {
|
|
114
|
+
const row = shard.byDecisionId(decisionId);
|
|
115
|
+
if (row) return row;
|
|
116
|
+
}
|
|
117
|
+
return undefined;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
byAgentRange(agentId: string, fromTs: string, toTs: string, limit = -1): IndexRow[] {
|
|
121
|
+
const fromDay = shardKeyForTs(fromTs);
|
|
122
|
+
const toDay = shardKeyForTs(toTs);
|
|
123
|
+
const rows: IndexRow[] = [];
|
|
124
|
+
for (const day of this.shardDays()) {
|
|
125
|
+
if (day < fromDay || day > toDay) continue;
|
|
126
|
+
const shard = this.shardForDay(day, false);
|
|
127
|
+
if (shard) rows.push(...shard.byAgentRange(agentId, fromTs, toTs, limit));
|
|
128
|
+
}
|
|
129
|
+
rows.sort((a, b) => (a.ts < b.ts ? -1 : a.ts > b.ts ? 1 : a.decisionId < b.decisionId ? -1 : 1));
|
|
130
|
+
return limit >= 0 ? rows.slice(0, limit) : rows;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
byAgentSchema(agentId: string, schemaHash: string, limit = -1): IndexRow[] {
|
|
134
|
+
const rows: IndexRow[] = [];
|
|
135
|
+
for (const shard of this.openShards()) {
|
|
136
|
+
rows.push(...shard.byAgentSchema(agentId, schemaHash, limit));
|
|
137
|
+
}
|
|
138
|
+
rows.sort((a, b) => (a.ts < b.ts ? -1 : a.ts > b.ts ? 1 : a.decisionId < b.decisionId ? -1 : 1));
|
|
139
|
+
return limit >= 0 ? rows.slice(0, limit) : rows;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/** Global chain tip for an agent across all shards (max decisionId). */
|
|
143
|
+
chainHead(agentId: string): IndexRow | undefined {
|
|
144
|
+
let head: IndexRow | undefined;
|
|
145
|
+
for (const shard of this.openShards()) {
|
|
146
|
+
const h = shard.chainHead(agentId);
|
|
147
|
+
if (h && (!head || h.decisionId > head.decisionId)) head = h;
|
|
148
|
+
}
|
|
149
|
+
return head;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
count(): number {
|
|
153
|
+
return this.openShards().reduce((n, s) => n + s.count(), 0);
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
close(): void {
|
|
157
|
+
for (const shard of this.shards.values()) shard.close();
|
|
158
|
+
this.shards.clear();
|
|
159
|
+
}
|
|
160
|
+
}
|