@openpond/evals 0.11.18 → 0.11.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (134) hide show
  1. package/dist/connected-evidence/contracts.js +14 -4
  2. package/dist/connected-evidence/imports.js +77 -17
  3. package/dist/connected-evidence/normalize.js +4 -3
  4. package/dist/connected-evidence/sources/claude-branches.js +52 -0
  5. package/dist/connected-evidence/sources/claude-code.js +7 -29
  6. package/dist/connected-evidence/sources/codex-authorization.js +40 -0
  7. package/dist/connected-evidence/sources/codex.js +7 -3
  8. package/dist/connected-evidence/sources/grok-build.js +137 -0
  9. package/dist/connected-evidence/sources/opencode.js +101 -0
  10. package/dist/connected-evidence/sources/pi.js +187 -0
  11. package/dist/graders.js +11 -1
  12. package/dist/learning/budgeted-reward-fixtures.js +1 -0
  13. package/dist/learning/contracts.js +10 -0
  14. package/dist/learning/index.js +1 -0
  15. package/dist/learning/model-judge.js +22 -5
  16. package/dist/learning/reward-calibration-closure.js +110 -0
  17. package/dist/learning/reward-calibration.js +20 -10
  18. package/dist/learning/reward-checks.js +1 -0
  19. package/dist/native-conversations/appimage-runtime.js +131 -0
  20. package/dist/native-conversations/collector-branches.js +79 -0
  21. package/dist/native-conversations/collector-contracts.js +9 -0
  22. package/dist/native-conversations/collector-destinations.js +27 -0
  23. package/dist/native-conversations/collector-errors.js +42 -0
  24. package/dist/native-conversations/collector-progress.js +114 -0
  25. package/dist/native-conversations/collector-store.js +247 -0
  26. package/dist/native-conversations/collector-sync-requests.js +52 -0
  27. package/dist/native-conversations/collector.js +322 -0
  28. package/dist/native-conversations/contracts.js +11 -0
  29. package/dist/native-conversations/database.js +149 -0
  30. package/dist/native-conversations/discovery.js +135 -0
  31. package/dist/native-conversations/history.js +298 -0
  32. package/dist/native-conversations/index.js +11 -0
  33. package/dist/native-conversations/location.js +29 -0
  34. package/dist/native-conversations/openclaw-cold-archive.js +91 -0
  35. package/dist/native-conversations/openclaw-database.js +130 -0
  36. package/dist/native-conversations/source-locations.js +112 -0
  37. package/dist/native-conversations/supervisor.js +208 -0
  38. package/dist/rewards.js +1 -1
  39. package/dist/types/connected-evidence/contracts.d.ts +15 -0
  40. package/dist/types/connected-evidence/contracts.d.ts.map +1 -1
  41. package/dist/types/connected-evidence/imports.d.ts +6 -0
  42. package/dist/types/connected-evidence/imports.d.ts.map +1 -1
  43. package/dist/types/connected-evidence/normalize.d.ts +4 -0
  44. package/dist/types/connected-evidence/normalize.d.ts.map +1 -1
  45. package/dist/types/connected-evidence/sources/claude-branches.d.ts +19 -0
  46. package/dist/types/connected-evidence/sources/claude-branches.d.ts.map +1 -0
  47. package/dist/types/connected-evidence/sources/claude-code.d.ts +5 -1
  48. package/dist/types/connected-evidence/sources/claude-code.d.ts.map +1 -1
  49. package/dist/types/connected-evidence/sources/codex-authorization.d.ts +18 -0
  50. package/dist/types/connected-evidence/sources/codex-authorization.d.ts.map +1 -0
  51. package/dist/types/connected-evidence/sources/codex.d.ts +5 -1
  52. package/dist/types/connected-evidence/sources/codex.d.ts.map +1 -1
  53. package/dist/types/connected-evidence/sources/grok-build.d.ts +79 -0
  54. package/dist/types/connected-evidence/sources/grok-build.d.ts.map +1 -0
  55. package/dist/types/connected-evidence/sources/history.d.ts +10 -2
  56. package/dist/types/connected-evidence/sources/history.d.ts.map +1 -1
  57. package/dist/types/connected-evidence/sources/opencode.d.ts +78 -0
  58. package/dist/types/connected-evidence/sources/opencode.d.ts.map +1 -0
  59. package/dist/types/connected-evidence/sources/pi.d.ts +79 -0
  60. package/dist/types/connected-evidence/sources/pi.d.ts.map +1 -0
  61. package/dist/types/graders.d.ts +81 -0
  62. package/dist/types/graders.d.ts.map +1 -1
  63. package/dist/types/learning/budgeted-reward-fixtures.d.ts +2 -1
  64. package/dist/types/learning/budgeted-reward-fixtures.d.ts.map +1 -1
  65. package/dist/types/learning/contracts.d.ts +167 -0
  66. package/dist/types/learning/contracts.d.ts.map +1 -1
  67. package/dist/types/learning/index.d.ts +1 -0
  68. package/dist/types/learning/index.d.ts.map +1 -1
  69. package/dist/types/learning/judge-budget.d.ts +3 -0
  70. package/dist/types/learning/judge-budget.d.ts.map +1 -1
  71. package/dist/types/learning/model-judge.d.ts +6 -1
  72. package/dist/types/learning/model-judge.d.ts.map +1 -1
  73. package/dist/types/learning/operations.d.ts +44 -0
  74. package/dist/types/learning/operations.d.ts.map +1 -1
  75. package/dist/types/learning/repository.d.ts +81 -0
  76. package/dist/types/learning/repository.d.ts.map +1 -1
  77. package/dist/types/learning/reward-calibration-closure.d.ts +2323 -0
  78. package/dist/types/learning/reward-calibration-closure.d.ts.map +1 -0
  79. package/dist/types/learning/reward-calibration.d.ts +1 -1
  80. package/dist/types/learning/reward-calibration.d.ts.map +1 -1
  81. package/dist/types/learning/reward-checks.d.ts +35 -0
  82. package/dist/types/learning/reward-checks.d.ts.map +1 -1
  83. package/dist/types/learning/transport.d.ts +103 -0
  84. package/dist/types/learning/transport.d.ts.map +1 -1
  85. package/dist/types/native-conversations/appimage-runtime.d.ts +12 -0
  86. package/dist/types/native-conversations/appimage-runtime.d.ts.map +1 -0
  87. package/dist/types/native-conversations/collector-branches.d.ts +22 -0
  88. package/dist/types/native-conversations/collector-branches.d.ts.map +1 -0
  89. package/dist/types/native-conversations/collector-contracts.d.ts +88 -0
  90. package/dist/types/native-conversations/collector-contracts.d.ts.map +1 -0
  91. package/dist/types/native-conversations/collector-destinations.d.ts +18 -0
  92. package/dist/types/native-conversations/collector-destinations.d.ts.map +1 -0
  93. package/dist/types/native-conversations/collector-errors.d.ts +13 -0
  94. package/dist/types/native-conversations/collector-errors.d.ts.map +1 -0
  95. package/dist/types/native-conversations/collector-progress.d.ts +24 -0
  96. package/dist/types/native-conversations/collector-progress.d.ts.map +1 -0
  97. package/dist/types/native-conversations/collector-store.d.ts +31 -0
  98. package/dist/types/native-conversations/collector-store.d.ts.map +1 -0
  99. package/dist/types/native-conversations/collector-sync-requests.d.ts +25 -0
  100. package/dist/types/native-conversations/collector-sync-requests.d.ts.map +1 -0
  101. package/dist/types/native-conversations/collector.d.ts +8 -0
  102. package/dist/types/native-conversations/collector.d.ts.map +1 -0
  103. package/dist/types/native-conversations/contracts.d.ts +47 -0
  104. package/dist/types/native-conversations/contracts.d.ts.map +1 -0
  105. package/dist/types/native-conversations/database.d.ts +14 -0
  106. package/dist/types/native-conversations/database.d.ts.map +1 -0
  107. package/dist/types/native-conversations/discovery.d.ts +9 -0
  108. package/dist/types/native-conversations/discovery.d.ts.map +1 -0
  109. package/dist/types/native-conversations/history.d.ts +23 -0
  110. package/dist/types/native-conversations/history.d.ts.map +1 -0
  111. package/dist/types/native-conversations/index.d.ts +12 -0
  112. package/dist/types/native-conversations/index.d.ts.map +1 -0
  113. package/dist/types/native-conversations/location.d.ts +3 -0
  114. package/dist/types/native-conversations/location.d.ts.map +1 -0
  115. package/dist/types/native-conversations/openclaw-cold-archive.d.ts +8 -0
  116. package/dist/types/native-conversations/openclaw-cold-archive.d.ts.map +1 -0
  117. package/dist/types/native-conversations/openclaw-database.d.ts +14 -0
  118. package/dist/types/native-conversations/openclaw-database.d.ts.map +1 -0
  119. package/dist/types/native-conversations/source-locations.d.ts +8 -0
  120. package/dist/types/native-conversations/source-locations.d.ts.map +1 -0
  121. package/dist/types/native-conversations/supervisor.d.ts +21 -0
  122. package/dist/types/native-conversations/supervisor.d.ts.map +1 -0
  123. package/dist/types/rewards.d.ts +50 -0
  124. package/dist/types/rewards.d.ts.map +1 -1
  125. package/dist/types/telemetry-analysis.d.ts +1 -1
  126. package/dist/types/telemetry-bundle.d.ts +2 -2
  127. package/dist/types/telemetry-catalog.d.ts +21 -21
  128. package/dist/types/telemetry.d.ts +2 -2
  129. package/package.json +11 -3
  130. package/schemas/learning/v1/command-request.schema.json +122 -91
  131. package/schemas/learning/v1/decision.schema.json +63 -0
  132. package/schemas/learning/v1/grade.schema.json +91 -0
  133. package/schemas/learning/v1/reward_check.schema.json +91 -0
  134. package/schemas/learning/v1/source.schema.json +28 -0
@@ -0,0 +1,79 @@
1
+ import { z } from "zod";
2
+ import { CollectorStore } from "./collector-store.js";
3
+ import { CollectorErrors } from "./collector-errors.js";
4
+ import { inspectSessionBranches, listSessions, readSession } from "./history.js";
5
+ const anchorSchema = z.object({ leafId: z.string().min(1).max(200), chainHash: z.string().regex(/^[a-f0-9]{64}$/), sourceInstanceId: z.string().min(1) }).strict();
6
+ const key = (connectionId, nativeSessionId) => `branch:${JSON.stringify([connectionId, nativeSessionId])}`;
7
+ export function collectorBranchAnchor(store, connectionId, nativeSessionId) {
8
+ const value = store.setting(key(connectionId, nativeSessionId));
9
+ if (!value)
10
+ return undefined;
11
+ const anchor = anchorSchema.parse(JSON.parse(value));
12
+ const connection = store.connections().find(item => item.id === connectionId);
13
+ return connection?.source.instanceId === anchor.sourceInstanceId ? { leafId: anchor.leafId, chainHash: anchor.chainHash } : undefined;
14
+ }
15
+ async function findSession(source, nativeSessionId) {
16
+ if (source.source !== "claude_code")
17
+ throw new Error("Branch selection is currently supported for Claude Code.");
18
+ let cursor;
19
+ const matches = [];
20
+ do {
21
+ const page = await listSessions(source, { cursor, limit: 100 });
22
+ matches.push(...page.items.filter(item => item.nativeSessionId === nativeSessionId));
23
+ cursor = page.nextCursor ?? undefined;
24
+ } while (cursor);
25
+ if (matches.length !== 1)
26
+ throw new Error("Select exactly one native session in this connection's source location.");
27
+ return matches[0];
28
+ }
29
+ export async function inspectCollectorBranches(directory, connectionId, nativeSessionId) {
30
+ const store = await CollectorStore.open(directory);
31
+ try {
32
+ const connection = store.connections().find(item => item.id === connectionId);
33
+ if (!connection)
34
+ throw new Error("Select a connection from openpond import status.");
35
+ const session = await findSession(connection.source, nativeSessionId);
36
+ return { session, ...await inspectSessionBranches(connection.source, session) };
37
+ }
38
+ finally {
39
+ store.close();
40
+ }
41
+ }
42
+ /** A paused connection fences earlier in-flight admission before narrowing its branch authority. */
43
+ export async function selectCollectorBranch(directory, connectionId, nativeSessionId, choice) {
44
+ const store = await CollectorStore.open(directory);
45
+ try {
46
+ const connection = store.connections().find(item => item.id === connectionId);
47
+ if (!connection || connection.state !== "paused")
48
+ throw new Error(`Pause this connection first: openpond import pause ${connectionId}`);
49
+ const session = await findSession(connection.source, nativeSessionId);
50
+ const inspected = await inspectSessionBranches(connection.source, session);
51
+ const branch = inspected.branches.find(item => item.leafId === choice.leafId);
52
+ if (inspected.revision !== choice.revision || !branch)
53
+ throw new Error("This conversation changed. Refresh its branches and choose again.");
54
+ await readSession(connection.source, session, { branchLeafId: choice.leafId, expectedBranchRevision: choice.revision });
55
+ store.database.exec("BEGIN IMMEDIATE");
56
+ try {
57
+ const current = store.connections().find(item => item.id === connectionId);
58
+ if (!current || current.state !== "paused" || current.revision !== connection.revision || current.source.instanceId !== connection.source.instanceId)
59
+ throw new Error("Connection changed during branch selection. Refresh and try again.");
60
+ store.set(key(connectionId, nativeSessionId), JSON.stringify({ leafId: branch.leafId, chainHash: branch.chainHash, sourceInstanceId: connection.source.instanceId }));
61
+ // Never admit previously queued sibling evidence after a changed choice.
62
+ for (const table of ["source_scans", "checkpoints", "pending"])
63
+ store.database.prepare(`DELETE FROM ${table} WHERE connection_id=?`).run(connectionId);
64
+ const errors = new CollectorErrors(store);
65
+ errors.set(connectionId, "source", errors.source(connectionId));
66
+ store.progress.reset(connectionId);
67
+ errors.set(connectionId, "admission", null);
68
+ store.database.exec("COMMIT");
69
+ }
70
+ catch (error) {
71
+ store.database.exec("ROLLBACK");
72
+ throw error;
73
+ }
74
+ return { connectionId, nativeSessionId, leafId: branch.leafId, state: "paused" };
75
+ }
76
+ finally {
77
+ store.close();
78
+ }
79
+ }
@@ -0,0 +1,9 @@
1
+ export const COLLECTOR_DEFAULTS = {
2
+ reconcileMs: 30000,
3
+ heartbeatMs: 10000,
4
+ staleMs: 60000,
5
+ maxPending: 10000,
6
+ maxBytes: 1024 * 1024 * 1024,
7
+ batchCases: 20,
8
+ batchBytes: 1024 * 1024,
9
+ };
@@ -0,0 +1,27 @@
1
+ /** Only destinations explicitly returned for this exact authenticated source. */
2
+ export function collectorDestinations(connection, remote) {
3
+ if (remote.id !== connection.id || remote.teamId !== connection.teamId ||
4
+ remote.projectId !== connection.projectId || remote.machineId !== connection.source.machineId ||
5
+ remote.sourceInstanceId !== connection.source.instanceId || remote.sourceRoot !== connection.source.root || remote.source !== connection.source.source)
6
+ throw new Error("The hosted destination belongs to another source or workspace.");
7
+ return { taskDatasetId: remote.taskDatasetId, conversationDatasetId: remote.conversationDatasetId };
8
+ }
9
+ export function collectorDestinationLinks(connection) {
10
+ const absent = { tasks: null, conversations: null };
11
+ if (!connection.accountBaseUrl || !connection.destinations)
12
+ return absent;
13
+ let origin;
14
+ try {
15
+ origin = new URL(connection.accountBaseUrl);
16
+ }
17
+ catch {
18
+ return absent;
19
+ }
20
+ if ((origin.protocol !== "https:" && !(origin.protocol === "http:" && ["localhost", "127.0.0.1"].includes(origin.hostname))) ||
21
+ origin.username || origin.password || origin.search || origin.hash || origin.pathname !== "/")
22
+ return absent;
23
+ const link = (id) => id
24
+ ? `${origin.origin}/console/datasets/${encodeURIComponent(id)}/tasks?project=${encodeURIComponent(connection.projectId)}&connection=${encodeURIComponent(connection.id)}`
25
+ : null;
26
+ return { tasks: link(connection.destinations.taskDatasetId), conversations: link(connection.destinations.conversationDatasetId) };
27
+ }
@@ -0,0 +1,42 @@
1
+ const phases = [
2
+ "source",
3
+ "admission",
4
+ "control",
5
+ "heartbeat",
6
+ ];
7
+ /** A recovered network probe cannot erase a durable unreadable-source or
8
+ * unacknowledged-admission failure, including after a collector restart. */
9
+ export class CollectorErrors {
10
+ store;
11
+ constructor(store) {
12
+ this.store = store;
13
+ }
14
+ read(id) {
15
+ const retained = this.store.setting(`errors:${id}`);
16
+ if (retained)
17
+ return JSON.parse(retained);
18
+ const source = this.store.database
19
+ .prepare("SELECT error FROM backfill_units WHERE connection_id=? AND state='error' AND error IS NOT NULL ORDER BY session_key LIMIT 1")
20
+ .get(id);
21
+ return typeof source?.error === "string" ? { source: source.error } : {};
22
+ }
23
+ message(id, exclude) {
24
+ const failures = this.read(id);
25
+ return (phases
26
+ .filter((phase) => phase !== exclude)
27
+ .map((phase) => failures[phase])
28
+ .find(Boolean) ?? null);
29
+ }
30
+ source(id) {
31
+ return this.read(id).source ?? null;
32
+ }
33
+ set(id, phase, message) {
34
+ const failures = this.read(id);
35
+ if (message)
36
+ failures[phase] = message.slice(0, 500);
37
+ else
38
+ delete failures[phase];
39
+ this.store.set(`errors:${id}`, JSON.stringify(failures));
40
+ this.store.error(id, this.message(id));
41
+ }
42
+ }
@@ -0,0 +1,114 @@
1
+ /** Frozen initial session selection; live arrivals never grow this denominator. */
2
+ export class CollectorProgress {
3
+ database;
4
+ constructor(database) {
5
+ this.database = database;
6
+ database.exec(`
7
+ CREATE TABLE IF NOT EXISTS backfills(connection_id TEXT PRIMARY KEY,discovered INTEGER NOT NULL DEFAULT 0);
8
+ CREATE TABLE IF NOT EXISTS backfill_units(connection_id TEXT NOT NULL,session_key TEXT NOT NULL,state TEXT NOT NULL,error TEXT,PRIMARY KEY(connection_id,session_key));
9
+ CREATE TABLE IF NOT EXISTS backfill_operations(operation_id TEXT PRIMARY KEY,connection_id TEXT NOT NULL,session_key TEXT NOT NULL,admitted INTEGER NOT NULL DEFAULT 0);
10
+ `);
11
+ }
12
+ reset(connectionId) {
13
+ for (const table of ["backfill_operations", "backfill_units", "backfills"])
14
+ this.database
15
+ .prepare(`DELETE FROM ${table} WHERE connection_id=?`)
16
+ .run(connectionId);
17
+ }
18
+ select(connectionId, sessionKey) {
19
+ this.database
20
+ .prepare("INSERT OR IGNORE INTO backfills(connection_id) VALUES(?)")
21
+ .run(connectionId);
22
+ const plan = this.database
23
+ .prepare("SELECT discovered FROM backfills WHERE connection_id=?")
24
+ .get(connectionId);
25
+ if (!plan?.discovered)
26
+ this.database
27
+ .prepare("INSERT OR IGNORE INTO backfill_units VALUES(?,?,'reading',NULL)")
28
+ .run(connectionId, sessionKey);
29
+ }
30
+ tracks(connectionId, sessionKey) {
31
+ const unit = this.database
32
+ .prepare("SELECT state FROM backfill_units WHERE connection_id=? AND session_key=?")
33
+ .get(connectionId, sessionKey);
34
+ return unit?.state === "reading" || unit?.state === "error";
35
+ }
36
+ queued(operationId, connectionId, sessionKey) {
37
+ if (this.tracks(connectionId, sessionKey))
38
+ this.database
39
+ .prepare("INSERT OR IGNORE INTO backfill_operations VALUES(?,?,?,0)")
40
+ .run(operationId, connectionId, sessionKey);
41
+ }
42
+ retained(connectionId, sessionKey, normalizedSessionKey) {
43
+ if (!this.tracks(connectionId, sessionKey))
44
+ return;
45
+ const pending = this.database
46
+ .prepare("SELECT id FROM pending WHERE connection_id=? AND json_extract(payload,'$.sessionKey')=?")
47
+ .all(connectionId, normalizedSessionKey);
48
+ for (const row of pending)
49
+ this.queued(String(row.id), connectionId, sessionKey);
50
+ }
51
+ read(connectionId, sessionKey, eligible) {
52
+ if (!this.tracks(connectionId, sessionKey))
53
+ return;
54
+ const pending = this.database
55
+ .prepare("SELECT COUNT(*) AS n FROM backfill_operations WHERE connection_id=? AND session_key=? AND admitted=0")
56
+ .get(connectionId, sessionKey);
57
+ this.database
58
+ .prepare("UPDATE backfill_units SET state=?,error=NULL WHERE connection_id=? AND session_key=?")
59
+ .run(Number(pending?.n) ? "queued" : eligible ? "complete" : "skipped", connectionId, sessionKey);
60
+ }
61
+ failed(connectionId, sessionKey, message) {
62
+ if (this.tracks(connectionId, sessionKey))
63
+ this.database
64
+ .prepare("UPDATE backfill_units SET state='error',error=? WHERE connection_id=? AND session_key=?")
65
+ .run(message.slice(0, 500), connectionId, sessionKey);
66
+ }
67
+ discovered(connectionId) {
68
+ this.database
69
+ .prepare("INSERT INTO backfills VALUES(?,1) ON CONFLICT(connection_id) DO UPDATE SET discovered=1")
70
+ .run(connectionId);
71
+ }
72
+ acknowledge(operationId) {
73
+ const operation = this.database
74
+ .prepare("SELECT connection_id,session_key FROM backfill_operations WHERE operation_id=?")
75
+ .get(operationId);
76
+ if (!operation)
77
+ return;
78
+ this.database
79
+ .prepare("UPDATE backfill_operations SET admitted=1 WHERE operation_id=?")
80
+ .run(operationId);
81
+ const pending = this.database
82
+ .prepare("SELECT COUNT(*) AS n FROM backfill_operations WHERE connection_id=? AND session_key=? AND admitted=0")
83
+ .get(operation.connection_id, operation.session_key);
84
+ if (!Number(pending?.n))
85
+ this.database
86
+ .prepare("UPDATE backfill_units SET state='complete' WHERE connection_id=? AND session_key=? AND state='queued'")
87
+ .run(operation.connection_id, operation.session_key);
88
+ }
89
+ status(connectionId) {
90
+ const plan = this.database
91
+ .prepare("SELECT discovered FROM backfills WHERE connection_id=?")
92
+ .get(connectionId);
93
+ const counts = this.database
94
+ .prepare("SELECT state,COUNT(*) AS n FROM backfill_units WHERE connection_id=? GROUP BY state")
95
+ .all(connectionId);
96
+ const count = (state) => Number(counts.find((row) => row.state === state)?.n ?? 0);
97
+ const total = counts.reduce((sum, row) => sum + Number(row.n), 0);
98
+ return {
99
+ stage: !plan?.discovered
100
+ ? "discovering"
101
+ : count("error")
102
+ ? "attention"
103
+ : count("reading")
104
+ ? "reading"
105
+ : count("queued")
106
+ ? "uploading"
107
+ : "complete",
108
+ total,
109
+ admitted: count("complete"),
110
+ skipped: count("skipped"),
111
+ failed: count("error"),
112
+ };
113
+ }
114
+ }
@@ -0,0 +1,247 @@
1
+ import { DatabaseSync } from "node:sqlite";
2
+ import { mkdir, chmod } from "node:fs/promises";
3
+ import { join } from "node:path";
4
+ import { COLLECTOR_DEFAULTS } from "./collector-contracts.js";
5
+ import { collectorDestinationLinks } from "./collector-destinations.js";
6
+ import { CollectorProgress } from "./collector-progress.js";
7
+ export class CollectorStore {
8
+ database;
9
+ progress;
10
+ constructor(database) {
11
+ this.database = database;
12
+ this.progress = new CollectorProgress(database);
13
+ }
14
+ static async open(directory) {
15
+ await mkdir(directory, { recursive: true, mode: 0o700 });
16
+ await chmod(directory, 0o700);
17
+ const file = join(directory, "collector.sqlite"), database = new DatabaseSync(file);
18
+ await chmod(file, 0o600);
19
+ database.exec(`PRAGMA journal_mode=WAL; PRAGMA synchronous=FULL; PRAGMA busy_timeout=1000;
20
+ CREATE TABLE IF NOT EXISTS settings(key TEXT PRIMARY KEY,value TEXT NOT NULL);
21
+ CREATE TABLE IF NOT EXISTS connections(id TEXT PRIMARY KEY,config TEXT NOT NULL,admitted INTEGER NOT NULL DEFAULT 0,error TEXT);
22
+ CREATE TABLE IF NOT EXISTS admitted_boundaries(connection_id TEXT NOT NULL,boundary_id TEXT NOT NULL,PRIMARY KEY(connection_id,boundary_id));
23
+ CREATE TABLE IF NOT EXISTS source_scans(connection_id TEXT NOT NULL,session_key TEXT NOT NULL,revision TEXT NOT NULL,PRIMARY KEY(connection_id,session_key));
24
+ CREATE TABLE IF NOT EXISTS checkpoints(connection_id TEXT NOT NULL,session_key TEXT NOT NULL,boundary_id TEXT NOT NULL,revision TEXT NOT NULL,PRIMARY KEY(connection_id,session_key,boundary_id));
25
+ CREATE TABLE IF NOT EXISTS pending(id TEXT PRIMARY KEY,connection_id TEXT NOT NULL,payload TEXT NOT NULL,bytes INTEGER NOT NULL,attempts INTEGER NOT NULL DEFAULT 0,retry_at INTEGER NOT NULL DEFAULT 0,created_at INTEGER NOT NULL);
26
+ INSERT OR IGNORE INTO settings VALUES('desiredState','stopped');`);
27
+ return new CollectorStore(database);
28
+ }
29
+ close() {
30
+ this.database.close();
31
+ }
32
+ setting(key) {
33
+ const row = this.database
34
+ .prepare("SELECT value FROM settings WHERE key=?")
35
+ .get(key);
36
+ return typeof row?.value === "string" ? row.value : null;
37
+ }
38
+ set(key, value) {
39
+ this.database
40
+ .prepare("INSERT INTO settings VALUES(?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value")
41
+ .run(key, value);
42
+ }
43
+ connections() {
44
+ return this.database
45
+ .prepare("SELECT config FROM connections ORDER BY id")
46
+ .all()
47
+ .map((row) => JSON.parse(String(row.config)));
48
+ }
49
+ put(connection) {
50
+ this.database.exec("BEGIN IMMEDIATE");
51
+ try {
52
+ const prior = this.connections().find((item) => item.id === connection.id);
53
+ if (prior && prior.revision > connection.revision) {
54
+ this.database.exec("COMMIT");
55
+ return;
56
+ }
57
+ const identityChanged = !!prior &&
58
+ (prior.source.instanceId !== connection.source.instanceId ||
59
+ prior.projectId !== connection.projectId ||
60
+ prior.teamId !== connection.teamId ||
61
+ prior.apiBaseUrl !== connection.apiBaseUrl ||
62
+ prior.account !== connection.account);
63
+ if (identityChanged)
64
+ connection = { ...connection, destinations: undefined };
65
+ if (prior && (identityChanged || prior.since !== connection.since)) {
66
+ // Scope changes fence queued evidence too: do not upload a wider old
67
+ // selection under newly narrowed consent. Reacquire the approved scope.
68
+ this.progress.reset(connection.id);
69
+ this.database
70
+ .prepare("DELETE FROM settings WHERE key=?")
71
+ .run(`errors:${connection.id}`);
72
+ this.database.prepare("DELETE FROM settings WHERE key=?").run(`syncAcknowledged:${connection.id}`);
73
+ connection = { ...connection, completedSyncRevision: 0 };
74
+ this.error(connection.id, null);
75
+ for (const table of ["source_scans", "checkpoints", "pending"])
76
+ this.database
77
+ .prepare(`DELETE FROM ${table} WHERE connection_id=?`)
78
+ .run(connection.id);
79
+ if (identityChanged) {
80
+ this.database
81
+ .prepare("DELETE FROM admitted_boundaries WHERE connection_id=?")
82
+ .run(connection.id);
83
+ this.database
84
+ .prepare("UPDATE connections SET admitted=0 WHERE id=?")
85
+ .run(connection.id);
86
+ this.database
87
+ .prepare("DELETE FROM settings WHERE key=?")
88
+ .run(`lastAdmission:${connection.id}`);
89
+ }
90
+ }
91
+ else if (prior) {
92
+ // UI/service writers may have read the connection before completion.
93
+ // Their later control write cannot erase a durable work receipt.
94
+ connection = {
95
+ ...connection,
96
+ requestedSyncRevision: Math.max(prior.requestedSyncRevision ?? 0, connection.requestedSyncRevision ?? 0),
97
+ completedSyncRevision: Math.max(prior.completedSyncRevision ?? 0, connection.completedSyncRevision ?? 0),
98
+ };
99
+ }
100
+ this.database
101
+ .prepare("INSERT INTO connections(id,config) VALUES(?,?) ON CONFLICT(id) DO UPDATE SET config=excluded.config")
102
+ .run(connection.id, JSON.stringify(connection));
103
+ this.database.exec("COMMIT");
104
+ }
105
+ catch (error) {
106
+ this.database.exec("ROLLBACK");
107
+ throw error;
108
+ }
109
+ }
110
+ error(id, message) {
111
+ this.database
112
+ .prepare("UPDATE connections SET error=? WHERE id=?")
113
+ .run(message?.slice(0, 500) ?? null, id);
114
+ }
115
+ queued(id) {
116
+ return Number((id
117
+ ? this.database
118
+ .prepare("SELECT COUNT(*) AS n FROM pending WHERE connection_id=?")
119
+ .get(id)
120
+ : this.database.prepare("SELECT COUNT(*) AS n FROM pending").get())?.n ?? 0);
121
+ }
122
+ sourceChanged(connectionId, sessionKey, revision) {
123
+ return (this.database
124
+ .prepare("SELECT revision FROM source_scans WHERE connection_id=? AND session_key=?")
125
+ .get(connectionId, sessionKey)?.revision !== revision);
126
+ }
127
+ scanned(connectionId, sessionKey, revision) {
128
+ this.database
129
+ .prepare("INSERT INTO source_scans VALUES(?,?,?) ON CONFLICT(connection_id,session_key) DO UPDATE SET revision=excluded.revision")
130
+ .run(connectionId, sessionKey, revision);
131
+ }
132
+ changed(connectionId, sessionKey, boundaryId, revision) {
133
+ return (this.database
134
+ .prepare("SELECT revision FROM checkpoints WHERE connection_id=? AND session_key=? AND boundary_id=?")
135
+ .get(connectionId, sessionKey, boundaryId)?.revision !== revision);
136
+ }
137
+ enqueue(entry, revisions, progressSessionKey) {
138
+ const payload = JSON.stringify(entry), bytes = Buffer.byteLength(payload);
139
+ this.database.exec("BEGIN IMMEDIATE");
140
+ try {
141
+ const usage = this.database
142
+ .prepare("SELECT COUNT(*) AS count,COALESCE(SUM(bytes),0) AS bytes FROM pending")
143
+ .get();
144
+ if (Number(usage.count) >= COLLECTOR_DEFAULTS.maxPending ||
145
+ Number(usage.bytes) + bytes > COLLECTOR_DEFAULTS.maxBytes)
146
+ throw new Error("Buffer full. Restore connectivity before acquiring more history.");
147
+ this.database
148
+ .prepare("INSERT OR IGNORE INTO pending(id,connection_id,payload,bytes,created_at) VALUES(?,?,?,?,?)")
149
+ .run(entry.operationId, entry.connectionId, payload, bytes, Date.now());
150
+ const update = this.database.prepare("INSERT INTO checkpoints VALUES(?,?,?,?) ON CONFLICT(connection_id,session_key,boundary_id) DO UPDATE SET revision=excluded.revision");
151
+ for (const boundary of revisions)
152
+ update.run(entry.connectionId, entry.sessionKey, boundary.id, boundary.revision);
153
+ if (progressSessionKey)
154
+ this.progress.queued(entry.operationId, entry.connectionId, progressSessionKey);
155
+ this.database.exec("COMMIT");
156
+ }
157
+ catch (error) {
158
+ this.database.exec("ROLLBACK");
159
+ throw error;
160
+ }
161
+ }
162
+ next(connectionId) {
163
+ const row = this.database
164
+ .prepare("SELECT payload,attempts FROM pending WHERE connection_id=? AND retry_at <= ? ORDER BY created_at,id LIMIT 1")
165
+ .get(connectionId, Date.now());
166
+ return row
167
+ ? {
168
+ entry: JSON.parse(String(row.payload)),
169
+ attempts: Number(row.attempts),
170
+ }
171
+ : null;
172
+ }
173
+ acknowledge(entry) {
174
+ this.database.exec("BEGIN IMMEDIATE");
175
+ try {
176
+ const removed = this.database
177
+ .prepare("DELETE FROM pending WHERE id=?")
178
+ .run(entry.operationId);
179
+ if (Number(removed.changes) > 0) {
180
+ this.progress.acknowledge(entry.operationId);
181
+ this.set(`lastAdmission:${entry.connectionId}`, new Date().toISOString());
182
+ let admitted = 0;
183
+ const insert = this.database.prepare("INSERT OR IGNORE INTO admitted_boundaries VALUES(?,?)");
184
+ for (const id of entry.boundaryIds)
185
+ admitted += Number(insert.run(entry.connectionId, id).changes);
186
+ this.database
187
+ .prepare("UPDATE connections SET admitted=admitted+? WHERE id=?")
188
+ .run(admitted, entry.connectionId);
189
+ }
190
+ this.database.exec("COMMIT");
191
+ }
192
+ catch (error) {
193
+ this.database.exec("ROLLBACK");
194
+ throw error;
195
+ }
196
+ }
197
+ retry(id, attempts) {
198
+ const delay = Math.min(60000, 1000 * 2 ** Math.min(attempts, 6));
199
+ this.database
200
+ .prepare("UPDATE pending SET attempts=?,retry_at=? WHERE id=?")
201
+ .run(attempts + 1, Date.now() + delay + Math.floor(Math.random() * Math.max(1, delay / 5)), id);
202
+ }
203
+ status() {
204
+ const heartbeatAt = this.setting("heartbeatAt"), pid = Number(this.setting("pid")) || null;
205
+ let alive = false;
206
+ try {
207
+ if (pid) {
208
+ process.kill(pid, 0);
209
+ alive = true;
210
+ }
211
+ }
212
+ catch { }
213
+ return {
214
+ schemaVersion: 1,
215
+ running: alive &&
216
+ !!heartbeatAt &&
217
+ Date.now() - Date.parse(heartbeatAt) < COLLECTOR_DEFAULTS.staleMs,
218
+ desiredState: this.setting("desiredState") === "running" ? "running" : "stopped",
219
+ pid,
220
+ heartbeatAt,
221
+ connections: this.database
222
+ .prepare("SELECT id,config,admitted,error FROM connections ORDER BY id")
223
+ .all()
224
+ .map((row) => {
225
+ const config = JSON.parse(String(row.config));
226
+ return {
227
+ id: String(row.id),
228
+ state: config.state,
229
+ projectId: config.projectId,
230
+ source: config.source.source,
231
+ queued: this.queued(String(row.id)),
232
+ admitted: Number(row.admitted),
233
+ error: typeof row.error === "string" ? row.error : null,
234
+ backfill: this.progress.status(String(row.id)),
235
+ pendingBytes: Number(this.database
236
+ .prepare("SELECT COALESCE(SUM(bytes),0) AS bytes FROM pending WHERE connection_id=?")
237
+ .get(String(row.id))?.bytes ?? 0),
238
+ lastAdmissionAt: this.setting(`lastAdmission:${row.id}`),
239
+ requestedSyncRevision: config.requestedSyncRevision ?? 0,
240
+ completedSyncRevision: config.completedSyncRevision ?? 0,
241
+ acknowledgedSyncRevision: Number(this.setting(`syncAcknowledged:${row.id}`)) || 0,
242
+ destinationLinks: collectorDestinationLinks(config),
243
+ };
244
+ }),
245
+ };
246
+ }
247
+ }
@@ -0,0 +1,52 @@
1
+ /** The request and completed-work receipt are durable. An in-process scan token
2
+ * deliberately is not: restart reconciles the retained source again before it
3
+ * can claim previously unfinished work completed. */
4
+ export class CollectorSyncRequests {
5
+ store;
6
+ acquisitions = new Map();
7
+ constructor(store) {
8
+ this.store = store;
9
+ }
10
+ observe(connection, remote) {
11
+ const requested = remote.requestedSyncRevision ?? connection.requestedSyncRevision ?? 0;
12
+ const completed = remote.completedSyncRevision ?? 0;
13
+ if (!Number.isSafeInteger(requested) || requested < 0 || requested > remote.revision ||
14
+ !Number.isSafeInteger(completed) || completed < 0 || completed > requested)
15
+ throw new Error("Invalid remote sync acknowledgement.");
16
+ const confirmed = this.confirmed(connection.id);
17
+ if (completed > confirmed)
18
+ this.store.set(`syncAcknowledged:${connection.id}`, String(completed));
19
+ return {
20
+ requestedSyncRevision: Math.max(connection.requestedSyncRevision ?? 0, requested),
21
+ completedSyncRevision: Math.max(connection.completedSyncRevision ?? 0, completed),
22
+ };
23
+ }
24
+ needsAcquisition(connection) {
25
+ const requested = connection.requestedSyncRevision ?? 0;
26
+ if (requested <= (connection.completedSyncRevision ?? 0))
27
+ return false;
28
+ const prior = this.acquisitions.get(connection.id);
29
+ return prior?.controlRevision !== connection.revision || prior.requestedSyncRevision !== requested;
30
+ }
31
+ acquired(id, acquisition) {
32
+ this.acquisitions.set(id, acquisition);
33
+ }
34
+ complete(connection, error) {
35
+ const acquired = this.acquisitions.get(connection.id);
36
+ const requested = connection.requestedSyncRevision ?? 0;
37
+ if (connection.state !== "active" || this.store.setting("desiredState") !== "running" ||
38
+ this.store.queued(connection.id) || error || !acquired?.succeeded ||
39
+ acquired.controlRevision !== connection.revision || acquired.requestedSyncRevision !== requested ||
40
+ requested <= (connection.completedSyncRevision ?? 0))
41
+ return connection;
42
+ const completed = { ...connection, completedSyncRevision: requested };
43
+ this.store.put(completed);
44
+ return completed;
45
+ }
46
+ confirmed(id) {
47
+ return Number(this.store.setting(`syncAcknowledged:${id}`)) || 0;
48
+ }
49
+ needsAcknowledgement(connection) {
50
+ return (connection.completedSyncRevision ?? 0) > this.confirmed(connection.id);
51
+ }
52
+ }