agents-can-communicate 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +46 -44
- package/bin/acc-update-worker.mjs +2 -1
- package/bin/entrypoints/hook-output.mjs +2 -0
- package/docs/ARCHITECTURE.md +19 -0
- package/docs/CAPABILITIES.md +1 -1
- package/docs/CLI.md +11 -1
- package/docs/CONFIGURATION.md +26 -0
- package/docs/GETTING_STARTED.md +4 -3
- package/docs/SECURITY_MODEL.md +6 -0
- package/docs/TROUBLESHOOTING.md +13 -0
- package/docs/UPGRADING.md +64 -0
- package/docs/WHY_ACC.md +32 -9
- package/docs/index.md +7 -5
- package/node_modules/@agents-can-communicate/adapter-antigravity/package.json +1 -1
- package/node_modules/@agents-can-communicate/adapter-antigravity/plugin/plugin.json +1 -1
- package/node_modules/@agents-can-communicate/adapter-antigravity/plugin/skills/acc/SKILL.md +3 -0
- package/node_modules/@agents-can-communicate/adapter-claude-code/package.json +1 -1
- package/node_modules/@agents-can-communicate/adapter-claude-code/plugin/skills/acc/SKILL.md +3 -0
- package/node_modules/@agents-can-communicate/adapter-claude-code/src/adapter.mjs +3 -1
- package/node_modules/@agents-can-communicate/adapter-claude-code/src/hooks.mjs +12 -0
- package/node_modules/@agents-can-communicate/adapter-codex/package.json +1 -1
- package/node_modules/@agents-can-communicate/adapter-codex/plugin/skills/acc/SKILL.md +3 -0
- package/node_modules/@agents-can-communicate/adapter-codex/src/adapter.mjs +2 -1
- package/node_modules/@agents-can-communicate/adapter-codex/src/allow-rule.mjs +129 -0
- package/node_modules/@agents-can-communicate/adapter-codex/src/hooks.mjs +11 -0
- package/node_modules/@agents-can-communicate/adapter-codex/src/install.mjs +18 -4
- package/node_modules/@agents-can-communicate/adapter-gemini-cli/extension/gemini-extension.json +1 -1
- package/node_modules/@agents-can-communicate/adapter-gemini-cli/extension/skills/acc/SKILL.md +3 -0
- package/node_modules/@agents-can-communicate/adapter-gemini-cli/package.json +1 -1
- package/node_modules/@agents-can-communicate/adapter-grok/package.json +1 -1
- package/node_modules/@agents-can-communicate/adapter-grok/plugin/skills/acc/SKILL.md +3 -0
- package/node_modules/@agents-can-communicate/adapter-kimi/package.json +1 -1
- package/node_modules/@agents-can-communicate/adapter-kimi/plugin/skills/acc/SKILL.md +3 -0
- package/node_modules/@agents-can-communicate/adapter-sdk/package.json +1 -1
- package/node_modules/@agents-can-communicate/adapter-sdk/src/capabilities.mjs +6 -0
- package/node_modules/@agents-can-communicate/adapter-sdk/src/own-version.mjs +27 -1
- package/node_modules/@agents-can-communicate/cli/package.json +1 -1
- package/node_modules/@agents-can-communicate/cli/src/args.mjs +4 -1
- package/node_modules/@agents-can-communicate/cli/src/doctor-command.mjs +14 -0
- package/node_modules/@agents-can-communicate/cli/src/help.mjs +2 -0
- package/node_modules/@agents-can-communicate/cli/src/main.mjs +12 -3
- package/node_modules/@agents-can-communicate/cli/src/managed-runtime/activation.mjs +19 -6
- package/node_modules/@agents-can-communicate/cli/src/managed-runtime/launchers.mjs +1 -1
- package/node_modules/@agents-can-communicate/cli/src/managed-runtime/portable-fs.mjs +41 -20
- package/node_modules/@agents-can-communicate/cli/src/managed-runtime/schedule.mjs +17 -1
- package/node_modules/@agents-can-communicate/cli/src/managed-runtime/store-upgrade.mjs +105 -0
- package/node_modules/@agents-can-communicate/cli/src/managed-runtime/worker.mjs +5 -0
- package/node_modules/@agents-can-communicate/cli/src/store-migration-command.mjs +39 -0
- package/node_modules/@agents-can-communicate/core/package.json +1 -1
- package/node_modules/@agents-can-communicate/core/src/conversations.mjs +21 -52
- package/node_modules/@agents-can-communicate/core/src/decision-changes.mjs +16 -11
- package/node_modules/@agents-can-communicate/core/src/inbox.mjs +12 -4
- package/node_modules/@agents-can-communicate/core/src/message-retry.mjs +43 -0
- package/node_modules/@agents-can-communicate/core/src/ports.mjs +12 -1
- package/node_modules/@agents-can-communicate/core/src/receipts.mjs +20 -6
- package/node_modules/@agents-can-communicate/delivery-router/package.json +1 -1
- package/node_modules/@agents-can-communicate/hook-runner/package.json +1 -1
- package/node_modules/@agents-can-communicate/hook-runner/src/runner.mjs +23 -3
- package/node_modules/@agents-can-communicate/installer/package.json +1 -1
- package/node_modules/@agents-can-communicate/installer/src/ownership.mjs +6 -2
- package/node_modules/@agents-can-communicate/mcp-server/package.json +1 -1
- package/node_modules/@agents-can-communicate/protocol/package.json +1 -1
- package/node_modules/@agents-can-communicate/protocol/src/index.mjs +1 -0
- package/node_modules/@agents-can-communicate/protocol/src/transaction-indexes.mjs +22 -0
- package/node_modules/@agents-can-communicate/storage-filesystem/package.json +1 -1
- package/node_modules/@agents-can-communicate/storage-filesystem/src/identity.mjs +22 -10
- package/node_modules/@agents-can-communicate/storage-filesystem/src/index-cache.mjs +144 -0
- package/node_modules/@agents-can-communicate/storage-filesystem/src/index-io.mjs +65 -0
- package/node_modules/@agents-can-communicate/storage-filesystem/src/index-pages.mjs +85 -0
- package/node_modules/@agents-can-communicate/storage-filesystem/src/index-reclaim-progress.mjs +35 -0
- package/node_modules/@agents-can-communicate/storage-filesystem/src/index-reclaim.mjs +142 -0
- package/node_modules/@agents-can-communicate/storage-filesystem/src/index-tree.mjs +142 -0
- package/node_modules/@agents-can-communicate/storage-filesystem/src/index.mjs +1 -0
- package/node_modules/@agents-can-communicate/storage-filesystem/src/journal.mjs +2 -0
- package/node_modules/@agents-can-communicate/storage-filesystem/src/portable-fs.mjs +41 -20
- package/node_modules/@agents-can-communicate/storage-filesystem/src/prune-durability.mjs +29 -0
- package/node_modules/@agents-can-communicate/storage-filesystem/src/reclaim.mjs +13 -3
- package/node_modules/@agents-can-communicate/storage-filesystem/src/recovery.mjs +9 -3
- package/node_modules/@agents-can-communicate/storage-filesystem/src/session-read.mjs +6 -4
- package/node_modules/@agents-can-communicate/storage-filesystem/src/state-reads.mjs +43 -0
- package/node_modules/@agents-can-communicate/storage-filesystem/src/store-migration.mjs +56 -0
- package/node_modules/@agents-can-communicate/storage-filesystem/src/store.mjs +61 -110
- package/node_modules/@agents-can-communicate/storage-filesystem/src/transaction-view.mjs +131 -0
- package/package.json +5 -4
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { AccError, EXIT, SCHEMA_VERSION, advanceReceipt }
|
|
2
2
|
from "@agents-can-communicate/protocol";
|
|
3
3
|
|
|
4
|
+
import { assertExactTransaction } from "./ports.mjs";
|
|
4
5
|
import { receiptId } from "./conversations.mjs";
|
|
5
6
|
|
|
6
7
|
// How long a live offer may stand with nothing after it before the next turn
|
|
@@ -78,6 +79,12 @@ function requireReceipt(tx, messageId, recipientParticipantId) {
|
|
|
78
79
|
return { message, receipt };
|
|
79
80
|
}
|
|
80
81
|
|
|
82
|
+
async function loadReceipt(tx, input) {
|
|
83
|
+
assertExactTransaction(tx);
|
|
84
|
+
await tx.load("receipt", receiptId(input.messageId, input.recipientParticipantId));
|
|
85
|
+
await tx.load("message", input.messageId);
|
|
86
|
+
}
|
|
87
|
+
|
|
81
88
|
function requireOfferableReceipt(tx, input, { allowRoomNextTurn }) {
|
|
82
89
|
const found = requireReceipt(tx, input.messageId, input.recipientParticipantId);
|
|
83
90
|
if (found.message.toParticipantIds.length === 0
|
|
@@ -105,8 +112,10 @@ export function createReceiptService(ports) {
|
|
|
105
112
|
const { store, clock, ids } = ports;
|
|
106
113
|
|
|
107
114
|
async function readReceipt(input) {
|
|
108
|
-
return store.transaction(tx =>
|
|
109
|
-
|
|
115
|
+
return store.transaction(async tx => {
|
|
116
|
+
await loadReceipt(tx, input);
|
|
117
|
+
return requireReceipt(tx, input.messageId, input.recipientParticipantId).receipt;
|
|
118
|
+
}, { kinds: ["message", "receipt"], exactKinds: ["message", "receipt"] });
|
|
110
119
|
}
|
|
111
120
|
|
|
112
121
|
async function recordOfferSucceeded(input) {
|
|
@@ -116,7 +125,9 @@ export function createReceiptService(ports) {
|
|
|
116
125
|
{ messageId: input.messageId, transport: input.transport });
|
|
117
126
|
}
|
|
118
127
|
const now = clock.now();
|
|
119
|
-
return store.transaction(tx => {
|
|
128
|
+
return store.transaction(async tx => {
|
|
129
|
+
await loadReceipt(tx, input);
|
|
130
|
+
await tx.load("session", input.targetSessionId);
|
|
120
131
|
const id = receiptId(input.messageId, input.recipientParticipantId);
|
|
121
132
|
const receipt = requireOfferableReceipt(tx, input, { allowRoomNextTurn: true });
|
|
122
133
|
const target = requireTarget(tx, input, { mustBeOpen: true });
|
|
@@ -153,7 +164,8 @@ export function createReceiptService(ports) {
|
|
|
153
164
|
tx.put("receipt", id, offered, tx.generationOf("receipt", id));
|
|
154
165
|
success({});
|
|
155
166
|
return offered;
|
|
156
|
-
}, { kinds: ["session", "message", "receipt"],
|
|
167
|
+
}, { kinds: ["session", "message", "receipt"],
|
|
168
|
+
exactKinds: ["session", "message", "receipt"], deadlineAt: input.deadlineAt });
|
|
157
169
|
}
|
|
158
170
|
|
|
159
171
|
async function recordOfferFailed(input) {
|
|
@@ -162,7 +174,9 @@ export function createReceiptService(ports) {
|
|
|
162
174
|
{ safeErrorCode: input.safeErrorCode });
|
|
163
175
|
}
|
|
164
176
|
const now = clock.now();
|
|
165
|
-
return store.transaction(tx => {
|
|
177
|
+
return store.transaction(async tx => {
|
|
178
|
+
await loadReceipt(tx, input);
|
|
179
|
+
await tx.load("session", input.targetSessionId);
|
|
166
180
|
const receipt = requireOfferableReceipt(tx, input, { allowRoomNextTurn: false });
|
|
167
181
|
const target = requireTarget(tx, input, { mustBeOpen: false });
|
|
168
182
|
return tx.append({ schemaVersion: SCHEMA_VERSION, eventId: ids.next("event"),
|
|
@@ -174,7 +188,7 @@ export function createReceiptService(ports) {
|
|
|
174
188
|
targetGeneration: target.generation,
|
|
175
189
|
transport: input.transport, adapterId: input.adapterId,
|
|
176
190
|
clientVersion: input.clientVersion, safeErrorCode: input.safeErrorCode } });
|
|
177
|
-
}, { kinds: ["session", "message", "receipt"] });
|
|
191
|
+
}, { kinds: ["session", "message", "receipt"], exactKinds: ["session", "message", "receipt"] });
|
|
178
192
|
}
|
|
179
193
|
|
|
180
194
|
return { readReceipt, recordOfferSucceeded, recordOfferFailed };
|
|
@@ -35,7 +35,7 @@ import { appendStartOwner, appendToolOwner, assertOwnerQuotable, ownerHeader, ow
|
|
|
35
35
|
const DEFAULT_BUDGET_MS = 5_000;
|
|
36
36
|
// Failures the hook output explains with advice of its own; any other error
|
|
37
37
|
// text stays out of what the client shows.
|
|
38
|
-
const KNOWN_FAILURES = new Set(["workspace_contains_runtime", "workspace_path_unquotable"]);
|
|
38
|
+
const KNOWN_FAILURES = new Set(["workspace_contains_runtime", "workspace_path_unquotable", "store_migration_required"]);
|
|
39
39
|
// Windows reads the table through WMI in Script Host, 145 ms on a
|
|
40
40
|
// windows-latest runner, and through PowerShell where Script Host is off, whose
|
|
41
41
|
// first start on a machine took 3.8 s. It gets more of the budget there, and
|
|
@@ -822,6 +822,7 @@ export async function runHook({ adapterId, payload, adapters, dataHome, env,
|
|
|
822
822
|
if (adapter === undefined) throw new Error(`no adapter named ${adapterId}`);
|
|
823
823
|
|
|
824
824
|
const event = await adapter.normalizeHook(payload, { args });
|
|
825
|
+
trail.kind = event.kind;
|
|
825
826
|
// A session start reads the process table once, and it depends on nothing
|
|
826
827
|
// below: it starts now, while the store opens. On Windows it is the part
|
|
827
828
|
// of a start that runs another program.
|
|
@@ -900,15 +901,34 @@ export async function runHook({ adapterId, payload, adapters, dataHome, env,
|
|
|
900
901
|
});
|
|
901
902
|
return await Promise.race([execute(), budget]);
|
|
902
903
|
} catch (error) {
|
|
904
|
+
const failureCode = error instanceof AccError && KNOWN_FAILURES.has(error.details?.reasonCode)
|
|
905
|
+
? error.details.reasonCode : null;
|
|
903
906
|
return { ...fallback, failed: true, reason: error.message,
|
|
904
|
-
...(
|
|
905
|
-
|
|
907
|
+
...(failureCode === null ? {} : { failureCode }),
|
|
908
|
+
...(failureCode === "store_migration_required"
|
|
909
|
+
? { stdout: migrationNotice(adapters?.[adapterId], trail.kind) } : {}),
|
|
906
910
|
...(Date.now() >= deadline ? { timedOut: true, phase: trail.phase } : {}) };
|
|
907
911
|
} finally {
|
|
908
912
|
clearTimeout(timer);
|
|
909
913
|
}
|
|
910
914
|
}
|
|
911
915
|
|
|
916
|
+
// What a person reads when their workspace's store waits for its migration: a
|
|
917
|
+
// client that shows hook output to the user says so, instead of the stderr line
|
|
918
|
+
// no client showed (2026-10-06). Never a hook input or error text.
|
|
919
|
+
const MIGRATION_NOTICE = "ACC: coordination is paused in this workspace until its store moves "
|
|
920
|
+
+ "to the new format. ACC does this by itself once older ACC clients have stopped; to do it "
|
|
921
|
+
+ "now, stop them and run: acc doctor --migrate-store";
|
|
922
|
+
function migrationNotice(adapter, kind) {
|
|
923
|
+
if (typeof adapter?.failOpenNotice !== "function" || typeof kind !== "string") return "";
|
|
924
|
+
try {
|
|
925
|
+
const rendered = adapter.failOpenNotice({ kind, message: MIGRATION_NOTICE });
|
|
926
|
+
return typeof rendered === "string" ? rendered : "";
|
|
927
|
+
} catch {
|
|
928
|
+
return "";
|
|
929
|
+
}
|
|
930
|
+
}
|
|
931
|
+
|
|
912
932
|
/**
|
|
913
933
|
* Register a client session before its first turn (#167).
|
|
914
934
|
*
|
|
@@ -29,6 +29,7 @@ export async function fingerprint(file) {
|
|
|
29
29
|
}
|
|
30
30
|
|
|
31
31
|
const CLIENT_ORPHAN_MARKER = ".orphaned_at";
|
|
32
|
+
const CLIENT_IN_USE_DIR = ".in_use";
|
|
32
33
|
|
|
33
34
|
/**
|
|
34
35
|
* Fingerprint of a whole directory ACC created.
|
|
@@ -50,8 +51,11 @@ export async function treeFingerprint(root) {
|
|
|
50
51
|
// Claude Code marks a plugin cache version its registry no longer names with
|
|
51
52
|
// `.orphaned_at`, and removes the marker again if the version comes back. That
|
|
52
53
|
// is the client's own bookkeeping inside a copy ACC keeps on purpose, never an
|
|
53
|
-
// edit, and ACC never writes a file by that name.
|
|
54
|
-
|
|
54
|
+
// edit, and ACC never writes a file by that name. A running session's
|
|
55
|
+
// `.in_use/<pid>` is the same kind of bookkeeping (2.1.286): after an update it
|
|
56
|
+
// stood in the previous version ACC keeps on purpose.
|
|
57
|
+
const files = entries.filter(entry => entry.isFile() && entry.name !== CLIENT_ORPHAN_MARKER
|
|
58
|
+
&& path.basename(entry.parentPath ?? entry.path) !== CLIENT_IN_USE_DIR)
|
|
55
59
|
.map(entry => path.relative(root, path.join(entry.parentPath ?? entry.path, entry.name)))
|
|
56
60
|
.sort();
|
|
57
61
|
const hash = createHash("sha256");
|
|
@@ -12,3 +12,4 @@ export { CONFIG_FILENAME, CONFIG_SCHEMA_VERSION, RUNTIME_KEYS, defaultProjectCon
|
|
|
12
12
|
export { RECEIPT_STATES, advanceReceipt } from "./states.mjs";
|
|
13
13
|
export { assertMatchableResource, normaliseResource } from "./resources.mjs";
|
|
14
14
|
export { assertDecisionChange } from "./decision-changes.mjs";
|
|
15
|
+
export { TRANSACTION_INDEXES, assertIndexTuple, indexKeysFor } from "./transaction-indexes.mjs";
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { AccError, EXIT } from "./errors.mjs";
|
|
2
|
+
import { assertPortableId } from "./ids.mjs";
|
|
3
|
+
|
|
4
|
+
export const TRANSACTION_INDEXES = Object.freeze({
|
|
5
|
+
messageByClientKey: Object.freeze({ kind: "message", fields: ["workspaceId", "fromParticipantId", "clientMessageId"] }),
|
|
6
|
+
receiptsByMessage: Object.freeze({ kind: "receipt", fields: ["workspaceId", "messageId"] }),
|
|
7
|
+
});
|
|
8
|
+
|
|
9
|
+
export function assertIndexTuple(index, tuple) {
|
|
10
|
+
const definition = Object.hasOwn(TRANSACTION_INDEXES, index) ? TRANSACTION_INDEXES[index] : null;
|
|
11
|
+
if (definition === null || !Array.isArray(tuple) || tuple.length !== definition.fields.length) {
|
|
12
|
+
throw new AccError(EXIT.DATA, "invalid transaction index tuple", { index });
|
|
13
|
+
}
|
|
14
|
+
tuple.forEach((value, position) => assertPortableId(value, definition.fields[position]));
|
|
15
|
+
return tuple;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export function indexKeysFor(kind, record) {
|
|
19
|
+
return Object.entries(TRANSACTION_INDEXES).filter(([, definition]) => definition.kind === kind)
|
|
20
|
+
.map(([index, definition]) => ({ index,
|
|
21
|
+
tuple: assertIndexTuple(index, definition.fields.map(field => record[field])) }));
|
|
22
|
+
}
|
|
@@ -4,17 +4,26 @@ import { AccError, EXIT, assertPortableId } from "@agents-can-communicate/protoc
|
|
|
4
4
|
|
|
5
5
|
import { encode, nameCommittedLater, publishAtomic, readJsonIfPresent } from "./atomic-json.mjs";
|
|
6
6
|
|
|
7
|
-
export const STORE_VERSION =
|
|
7
|
+
export const STORE_VERSION = 7;
|
|
8
8
|
|
|
9
9
|
export function identityPath(paths) {
|
|
10
10
|
return path.join(paths.root, "protocol.json");
|
|
11
11
|
}
|
|
12
12
|
|
|
13
|
-
function assertIdentity(record, workspaceId, filePath) {
|
|
14
|
-
if (record?.storeVersion
|
|
15
|
-
|
|
16
|
-
|
|
13
|
+
function assertIdentity(record, workspaceId, filePath, versions = [STORE_VERSION]) {
|
|
14
|
+
if (!versions.includes(record?.storeVersion)) {
|
|
15
|
+
const message = record?.storeVersion === 6
|
|
16
|
+
? "store contract 6 requires explicit migration: stop other store users and run acc doctor --migrate-store"
|
|
17
|
+
: "unknown store version";
|
|
18
|
+
throw new AccError(EXIT.DATA, message, { filePath,
|
|
19
|
+
storeVersion: record?.storeVersion,
|
|
20
|
+
...(record?.storeVersion === 6 ? { reasonCode: "store_migration_required" } : {}) });
|
|
17
21
|
}
|
|
22
|
+
if (Object.keys(record).sort().join(",") !== "initialisedAt,storeVersion,workspaceId"
|
|
23
|
+
|| typeof record.initialisedAt !== "string" || !Number.isFinite(Date.parse(record.initialisedAt))) {
|
|
24
|
+
throw new AccError(EXIT.DATA, "invalid store identity", { filePath });
|
|
25
|
+
}
|
|
26
|
+
assertPortableId(record.workspaceId, "store workspace id");
|
|
18
27
|
if (record.workspaceId !== workspaceId) {
|
|
19
28
|
throw new AccError(EXIT.DATA, "store belongs to a different workspace",
|
|
20
29
|
{ filePath, expected: workspaceId, actual: record.workspaceId });
|
|
@@ -63,9 +72,12 @@ export async function requireStoreIdentity(paths, { workspaceId, clock, create =
|
|
|
63
72
|
export async function readStoreIdentity(paths) {
|
|
64
73
|
const found = await readJsonIfPresent(identityPath(paths), paths.root);
|
|
65
74
|
if (found === null) return null;
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
75
|
+
return assertIdentity(found.value, found.value?.workspaceId, identityPath(paths));
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
export async function readMigrationIdentity(paths, workspaceId) {
|
|
79
|
+
assertPortableId(workspaceId, "workspace id");
|
|
80
|
+
const found = await readJsonIfPresent(identityPath(paths), paths.root);
|
|
81
|
+
if (found === null) throw new AccError(EXIT.DATA, "store is not initialised", { filePath: identityPath(paths) });
|
|
82
|
+
return assertIdentity(found.value, workspaceId, identityPath(paths), [6, STORE_VERSION]);
|
|
71
83
|
}
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
import { AccError, EXIT, TRANSACTION_INDEXES, assertIndexTuple, indexKeysFor }
|
|
2
|
+
from "@agents-can-communicate/protocol";
|
|
3
|
+
import { assertPublicationDeadline } from "./deadline.mjs";
|
|
4
|
+
import { createIndexIO } from "./index-io.mjs";
|
|
5
|
+
import { createIndexTree } from "./index-tree.mjs";
|
|
6
|
+
import { encodeIndexPage, IndexCacheUnavailable } from "./index-pages.mjs";
|
|
7
|
+
|
|
8
|
+
const same = (a, b) => JSON.stringify(a) === JSON.stringify(b);
|
|
9
|
+
const matches = (kind, record, index, tuple) => indexKeysFor(kind, record)
|
|
10
|
+
.some(key => key.index === index && same(key.tuple, tuple));
|
|
11
|
+
const references = page => page.type === "branch" ? page.children.map(([, hash]) => hash)
|
|
12
|
+
: page.type === "leaf" ? [...page.entries.map(entry => entry.ids), page.next].filter(Boolean)
|
|
13
|
+
: [page.next].filter(Boolean);
|
|
14
|
+
|
|
15
|
+
export function createIndexCache({ paths, root, workspaceId, publishOptions, loadPrimary }) {
|
|
16
|
+
let diagnostic = [];
|
|
17
|
+
const note = (code, reason) => { diagnostic = [{ code, reason }]; };
|
|
18
|
+
const tools = deadlineAt => {
|
|
19
|
+
const io = createIndexIO({ paths, root, publishOptions: { ...publishOptions, deadlineAt } });
|
|
20
|
+
const bounded = Object.fromEntries(Object.entries(io).map(([name, operation]) => [name, (...args) => {
|
|
21
|
+
assertPublicationDeadline(deadlineAt); return operation(...args);
|
|
22
|
+
}]));
|
|
23
|
+
return { io: bounded, tree: createIndexTree(bounded) };
|
|
24
|
+
};
|
|
25
|
+
async function valid(io, active) {
|
|
26
|
+
const manifest = await io.readManifest();
|
|
27
|
+
if (manifest === null) return null;
|
|
28
|
+
if (active.state !== "idle" || manifest.workspaceId !== workspaceId
|
|
29
|
+
|| manifest.journalGeneration !== active.generation) throw new IndexCacheUnavailable("stale_manifest");
|
|
30
|
+
// Descendants are validated on their selected route. A root fault cannot
|
|
31
|
+
// be carried across an unrelated writer by simply changing its epoch.
|
|
32
|
+
for (const hash of Object.values(manifest.roots)) if (hash !== null) await io.readPage(hash);
|
|
33
|
+
return manifest;
|
|
34
|
+
}
|
|
35
|
+
async function phase(name) {
|
|
36
|
+
try { await publishOptions.failAt?.(name); }
|
|
37
|
+
catch (error) { throw new IndexCacheUnavailable("cache_publication_failed", error); }
|
|
38
|
+
}
|
|
39
|
+
async function publishManifest(io, manifest) {
|
|
40
|
+
await phase("after-index-pages");
|
|
41
|
+
await phase("before-index-manifest");
|
|
42
|
+
await io.writeManifest(manifest);
|
|
43
|
+
}
|
|
44
|
+
async function rebuild(active, deadlineAt, { required = false } = {}) {
|
|
45
|
+
// Authoritative reads stay outside every cache-error catch.
|
|
46
|
+
const primaries = await loadPrimary({ deadlineAt });
|
|
47
|
+
const deltas = Object.fromEntries(Object.keys(TRANSACTION_INDEXES).map(index => [index, []]));
|
|
48
|
+
for (const entry of primaries.values()) {
|
|
49
|
+
assertPublicationDeadline(deadlineAt);
|
|
50
|
+
if (entry.record.workspaceId !== workspaceId) {
|
|
51
|
+
throw new AccError(EXIT.DATA, "indexed primary belongs to a different workspace");
|
|
52
|
+
}
|
|
53
|
+
for (const key of indexKeysFor(entry.kind, entry.record)) deltas[key.index].push({ ...key, id: entry.id, operation: "put" });
|
|
54
|
+
}
|
|
55
|
+
const pages = new Map();
|
|
56
|
+
const tree = createIndexTree({ readPage: async hash => pages.get(hash), writePage: async page => {
|
|
57
|
+
assertPublicationDeadline(deadlineAt);
|
|
58
|
+
const { hash } = encodeIndexPage(page); pages.set(hash, page); return hash;
|
|
59
|
+
} });
|
|
60
|
+
const roots = {};
|
|
61
|
+
for (const index of Object.keys(deltas)) roots[index] = await tree.apply(null, deltas[index]);
|
|
62
|
+
const manifest = { indexVersion: 1, workspaceId, journalGeneration: active.generation, roots };
|
|
63
|
+
const { io } = tools(deadlineAt);
|
|
64
|
+
try {
|
|
65
|
+
// Publish only the final reachable tree, children before parents. The
|
|
66
|
+
// path copies made during a rebuild need never reach the filesystem.
|
|
67
|
+
const published = new Set();
|
|
68
|
+
async function publish(hash) {
|
|
69
|
+
if (hash === null || published.has(hash)) return;
|
|
70
|
+
const page = pages.get(hash);
|
|
71
|
+
for (const child of references(page)) await publish(child);
|
|
72
|
+
await io.writePage(page); published.add(hash);
|
|
73
|
+
}
|
|
74
|
+
for (const hash of Object.values(roots)) await publish(hash);
|
|
75
|
+
await publishManifest(io, manifest);
|
|
76
|
+
note("index_rebuilt", "verified_primary");
|
|
77
|
+
} catch (error) {
|
|
78
|
+
if (!(error instanceof IndexCacheUnavailable)) throw error;
|
|
79
|
+
note("index_unavailable", error.reason);
|
|
80
|
+
if (required) throw error;
|
|
81
|
+
}
|
|
82
|
+
return { manifest, tree };
|
|
83
|
+
}
|
|
84
|
+
async function lookup(index, tuple, { active, staged, deadlineAt, verify }) {
|
|
85
|
+
assertIndexTuple(index, tuple);
|
|
86
|
+
let tree, manifest;
|
|
87
|
+
try {
|
|
88
|
+
const selected = tools(deadlineAt);
|
|
89
|
+
tree = selected.tree;
|
|
90
|
+
manifest = await valid(selected.io, active);
|
|
91
|
+
if (manifest === null) throw new IndexCacheUnavailable("missing_manifest");
|
|
92
|
+
return await select(await tree.lookup(manifest.roots[index], tuple));
|
|
93
|
+
} catch (error) {
|
|
94
|
+
if (!(error instanceof IndexCacheUnavailable)) throw error;
|
|
95
|
+
}
|
|
96
|
+
({ manifest, tree } = await rebuild(active, deadlineAt));
|
|
97
|
+
return select(await tree.lookup(manifest.roots[index], tuple));
|
|
98
|
+
|
|
99
|
+
async function select(ids) {
|
|
100
|
+
const selected = new Set(ids), kind = TRANSACTION_INDEXES[index].kind;
|
|
101
|
+
for (const entry of staged.values()) {
|
|
102
|
+
if (entry.kind !== kind) continue;
|
|
103
|
+
selected.delete(entry.id);
|
|
104
|
+
if (entry.removed !== true && matches(kind, entry.record, index, tuple)) selected.add(entry.id);
|
|
105
|
+
}
|
|
106
|
+
const result = [];
|
|
107
|
+
// Match eager listJsonFiles ordering, including prefix IDs such as a/a-b.
|
|
108
|
+
for (const id of [...selected].sort((a, b) => `${a}.json`.localeCompare(`${b}.json`))) {
|
|
109
|
+
assertPublicationDeadline(deadlineAt);
|
|
110
|
+
if (await verify(id)) result.push(id);
|
|
111
|
+
}
|
|
112
|
+
return result;
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
function prepareDeltas(loaded, staged) {
|
|
116
|
+
const deltas = [];
|
|
117
|
+
for (const [id, next] of staged) {
|
|
118
|
+
const previous = loaded.get(id);
|
|
119
|
+
const oldKeys = previous == null ? [] : indexKeysFor(previous.kind, previous.record);
|
|
120
|
+
const newKeys = next.removed === true ? [] : indexKeysFor(next.kind, next.record);
|
|
121
|
+
for (const key of oldKeys) if (!newKeys.some(found => same(found, key))) deltas.push({ ...key, id: next.id, operation: "remove" });
|
|
122
|
+
for (const key of newKeys) if (!oldKeys.some(found => same(found, key))) deltas.push({ ...key, id: next.id, operation: "put" });
|
|
123
|
+
}
|
|
124
|
+
return deltas;
|
|
125
|
+
}
|
|
126
|
+
async function commit({ before, after, deltas, deadlineAt }) {
|
|
127
|
+
try {
|
|
128
|
+
await phase("after-primary-commit");
|
|
129
|
+
const { io, tree } = tools(deadlineAt), manifest = await valid(io, before);
|
|
130
|
+
if (manifest === null) return; // An unrelated writer never constructs a cache.
|
|
131
|
+
const roots = { ...manifest.roots };
|
|
132
|
+
for (const index of Object.keys(roots)) {
|
|
133
|
+
roots[index] = await tree.apply(roots[index], deltas.filter(delta => delta.index === index));
|
|
134
|
+
}
|
|
135
|
+
await publishManifest(io, { ...manifest, roots, journalGeneration: after.generation });
|
|
136
|
+
} catch (error) {
|
|
137
|
+
if (!(error instanceof IndexCacheUnavailable)
|
|
138
|
+
&& !(error.code === EXIT.CONFLICT && Date.now() >= deadlineAt)) throw error;
|
|
139
|
+
note("index_unavailable", error instanceof IndexCacheUnavailable ? error.reason : "deadline_expired");
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
return Object.freeze({ lookup, prepareDeltas, commit, rebuild,
|
|
143
|
+
diagnostics: () => diagnostic.map(item => ({ ...item })) });
|
|
144
|
+
}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import path from "node:path";
|
|
2
|
+
import { assertPortableId } from "@agents-can-communicate/protocol";
|
|
3
|
+
import { publishAtomic } from "./atomic-json.mjs";
|
|
4
|
+
import { withRegularNoFollow } from "./safe-file.mjs";
|
|
5
|
+
import { assertPageHash, decodeIndexPage, encodeIndexPage, INDEX_PAGE_BYTES, IndexCacheUnavailable }
|
|
6
|
+
from "./index-pages.mjs";
|
|
7
|
+
|
|
8
|
+
function manifestShape(value) {
|
|
9
|
+
const keys = object => object && !Array.isArray(object) ? Object.keys(object).sort().join(",") : "";
|
|
10
|
+
if (keys(value) !== "indexVersion,journalGeneration,roots,workspaceId"
|
|
11
|
+
|| value.indexVersion !== 1 || !/^[0-9]{16}$/.test(value.journalGeneration)
|
|
12
|
+
|| keys(value.roots) !== "messageByClientKey,receiptsByMessage") {
|
|
13
|
+
throw new IndexCacheUnavailable("invalid_manifest");
|
|
14
|
+
}
|
|
15
|
+
assertPortableId(value.workspaceId, "index workspace ID");
|
|
16
|
+
for (const hash of Object.values(value.roots)) if (hash !== null) assertPageHash(hash);
|
|
17
|
+
return value;
|
|
18
|
+
}
|
|
19
|
+
export function createIndexIO({ paths, root, publishOptions }) {
|
|
20
|
+
const directory = path.join(root, "indexes", "v1");
|
|
21
|
+
const options = { ...publishOptions, root, tmpDir: paths.tmp, stageDir: paths.stage, durability: "none" };
|
|
22
|
+
const cache = async operation => {
|
|
23
|
+
try { return await operation(); }
|
|
24
|
+
catch (error) {
|
|
25
|
+
if (error instanceof IndexCacheUnavailable) throw error;
|
|
26
|
+
throw new IndexCacheUnavailable(error.code === "ENOENT" ? "missing_page" : "unsafe_cache_io", error);
|
|
27
|
+
}
|
|
28
|
+
};
|
|
29
|
+
const readBytes = async file => {
|
|
30
|
+
const bytes = await withRegularNoFollow(file, root, "r", (handle, stat) =>
|
|
31
|
+
stat.size > INDEX_PAGE_BYTES ? null : handle.readFile());
|
|
32
|
+
if (bytes === null) throw new IndexCacheUnavailable("page_byte_limit");
|
|
33
|
+
return bytes;
|
|
34
|
+
};
|
|
35
|
+
return Object.freeze({
|
|
36
|
+
readPage: hash => cache(async () => decodeIndexPage(assertPageHash(hash),
|
|
37
|
+
await readBytes(path.join(directory, "pages", hash + ".json")))),
|
|
38
|
+
writePage: page => cache(async () => {
|
|
39
|
+
const { hash, bytes } = encodeIndexPage(page);
|
|
40
|
+
const file = path.join(directory, "pages", hash + ".json");
|
|
41
|
+
let damaged = false;
|
|
42
|
+
try { damaged = !(await readBytes(file)).equals(bytes); }
|
|
43
|
+
catch (error) {
|
|
44
|
+
if (error instanceof IndexCacheUnavailable && error.reason === "page_byte_limit") damaged = true;
|
|
45
|
+
else if (error.code !== "ENOENT") throw error;
|
|
46
|
+
}
|
|
47
|
+
// Valid pages stay immutable. A safely opened but damaged cache file is
|
|
48
|
+
// replaced with the verified bytes for its existing content address.
|
|
49
|
+
await publishAtomic(file, bytes, { ...options, replace: damaged });
|
|
50
|
+
return hash;
|
|
51
|
+
}),
|
|
52
|
+
readManifest: () => cache(async () => {
|
|
53
|
+
let bytes;
|
|
54
|
+
try { bytes = await readBytes(path.join(directory, "cache.json")); }
|
|
55
|
+
catch (error) { if (error.code === "ENOENT") return null; throw error; }
|
|
56
|
+
try { return manifestShape(JSON.parse(bytes.toString("utf8"))); }
|
|
57
|
+
catch (error) { throw new IndexCacheUnavailable("invalid_manifest", error); }
|
|
58
|
+
}),
|
|
59
|
+
writeManifest: manifest => cache(async () => {
|
|
60
|
+
manifestShape(manifest);
|
|
61
|
+
await publishAtomic(path.join(directory, "cache.json"),
|
|
62
|
+
Buffer.from(JSON.stringify(manifest) + "\n"), { ...options, replace: true });
|
|
63
|
+
}),
|
|
64
|
+
});
|
|
65
|
+
}
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { assertPortableId } from "@agents-can-communicate/protocol";
|
|
3
|
+
|
|
4
|
+
export const INDEX_PAGE_BYTES = 65_536;
|
|
5
|
+
export const INDEX_PAGE_ENTRIES = 32;
|
|
6
|
+
const HASH = /^[a-f0-9]{64}$/;
|
|
7
|
+
export class IndexCacheUnavailable extends Error {
|
|
8
|
+
constructor(reason, cause) {
|
|
9
|
+
super(`transaction index unavailable: ${reason}`, { cause });
|
|
10
|
+
this.name = "IndexCacheUnavailable";
|
|
11
|
+
this.reason = reason;
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
const refuse = reason => { throw new IndexCacheUnavailable(reason); };
|
|
15
|
+
export function assertPageHash(hash) {
|
|
16
|
+
if (typeof hash !== "string" || !HASH.test(hash)) refuse("invalid_reference");
|
|
17
|
+
return hash;
|
|
18
|
+
}
|
|
19
|
+
function closed(value, keys) {
|
|
20
|
+
if (value === null || typeof value !== "object" || Array.isArray(value)
|
|
21
|
+
|| Object.keys(value).sort().join(",") !== [...keys].sort().join(",")) refuse("invalid_page_shape");
|
|
22
|
+
}
|
|
23
|
+
function unique(values) {
|
|
24
|
+
if (new Set(values).size !== values.length) refuse("duplicate_page_entry");
|
|
25
|
+
}
|
|
26
|
+
export function canonicalTuple(tuple) {
|
|
27
|
+
if (!Array.isArray(tuple) || ![2, 3].includes(tuple.length)) refuse("invalid_tuple");
|
|
28
|
+
try { tuple.forEach(id => assertPortableId(id, "index tuple member")); }
|
|
29
|
+
catch (error) { throw new IndexCacheUnavailable("invalid_tuple", error); }
|
|
30
|
+
return JSON.stringify(tuple);
|
|
31
|
+
}
|
|
32
|
+
function limited(values, limit = INDEX_PAGE_ENTRIES) {
|
|
33
|
+
if (!Array.isArray(values) || values.length === 0 || values.length > limit) refuse("page_entry_limit");
|
|
34
|
+
}
|
|
35
|
+
export function validateIndexPage(page) {
|
|
36
|
+
if (page?.type === "branch") {
|
|
37
|
+
closed(page, ["type", "children"]); limited(page.children, 16);
|
|
38
|
+
const routes = [];
|
|
39
|
+
for (const child of page.children) {
|
|
40
|
+
if (!Array.isArray(child) || child.length !== 2 || !/^[a-f0-9]$/.test(child[0])) refuse("invalid_child");
|
|
41
|
+
assertPageHash(child[1]); routes.push(child[0]);
|
|
42
|
+
}
|
|
43
|
+
unique(routes);
|
|
44
|
+
if (routes.join("") !== [...routes].sort().join("")) refuse("noncanonical_routes");
|
|
45
|
+
} else if (page?.type === "leaf") {
|
|
46
|
+
closed(page, ["type", "entries", "next"]); limited(page.entries);
|
|
47
|
+
if (page.next !== null) assertPageHash(page.next);
|
|
48
|
+
const keys = [];
|
|
49
|
+
for (const entry of page.entries) {
|
|
50
|
+
closed(entry, ["digest", "tuple", "ids"]);
|
|
51
|
+
assertPageHash(entry.digest); assertPageHash(entry.ids); keys.push(canonicalTuple(entry.tuple));
|
|
52
|
+
}
|
|
53
|
+
unique(keys);
|
|
54
|
+
if (keys.some((key, i) => i > 0 && keys[i - 1].localeCompare(key) >= 0)) refuse("noncanonical_keys");
|
|
55
|
+
} else if (page?.type === "group") {
|
|
56
|
+
closed(page, ["type", "ids", "next"]); limited(page.ids);
|
|
57
|
+
if (page.next !== null) assertPageHash(page.next);
|
|
58
|
+
try { page.ids.forEach(id => assertPortableId(id, "index record ID")); }
|
|
59
|
+
catch (error) { throw new IndexCacheUnavailable("invalid_record_id", error); }
|
|
60
|
+
unique(page.ids);
|
|
61
|
+
if (page.ids.some((id, i) => i > 0 && page.ids[i - 1].localeCompare(id) >= 0)) refuse("noncanonical_ids");
|
|
62
|
+
} else refuse("unknown_page_type");
|
|
63
|
+
return page;
|
|
64
|
+
}
|
|
65
|
+
export function encodeIndexPage(page) {
|
|
66
|
+
validateIndexPage(page);
|
|
67
|
+
// Fixed field order makes insertion order irrelevant to content addressing.
|
|
68
|
+
const canonical = page.type === "branch" ? { type: page.type, children: page.children }
|
|
69
|
+
: page.type === "group" ? { type: page.type, ids: page.ids, next: page.next }
|
|
70
|
+
: { type: page.type, entries: page.entries.map(({ digest, tuple, ids }) => ({ digest, tuple, ids })), next: page.next };
|
|
71
|
+
const bytes = Buffer.from(JSON.stringify(canonical) + "\n");
|
|
72
|
+
if (bytes.length > INDEX_PAGE_BYTES) refuse("page_byte_limit");
|
|
73
|
+
return { bytes, hash: createHash("sha256").update(bytes).digest("hex") };
|
|
74
|
+
}
|
|
75
|
+
export function decodeIndexPage(hash, bytes) {
|
|
76
|
+
assertPageHash(hash);
|
|
77
|
+
if (!Buffer.isBuffer(bytes) || bytes.length > INDEX_PAGE_BYTES) refuse("page_byte_limit");
|
|
78
|
+
let page;
|
|
79
|
+
try { page = JSON.parse(bytes.toString("utf8")); }
|
|
80
|
+
catch (error) { throw new IndexCacheUnavailable("invalid_page_json", error); }
|
|
81
|
+
const encoded = encodeIndexPage(page);
|
|
82
|
+
if (!encoded.bytes.equals(bytes)) refuse("noncanonical_bytes");
|
|
83
|
+
if (encoded.hash !== hash) refuse("page_hash_mismatch");
|
|
84
|
+
return page;
|
|
85
|
+
}
|
package/node_modules/@agents-can-communicate/storage-filesystem/src/index-reclaim-progress.mjs
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { publishAtomic } from "./atomic-json.mjs";
|
|
4
|
+
import { withRegularNoFollow } from "./safe-file.mjs";
|
|
5
|
+
|
|
6
|
+
const hash = value => typeof value === "string" && /^[a-f0-9]{64}$/.test(value);
|
|
7
|
+
const digest = bytes => createHash("sha256").update(bytes).digest("hex");
|
|
8
|
+
const file = root => path.join(root, "indexes/v1/reclaim.json");
|
|
9
|
+
const fresh = () => ({ version: 1, pending: [], visited: [], cursor: null, done: false });
|
|
10
|
+
const valid = state => state && Object.keys(state).sort().join(",") === "cursor,done,pending,version,visited"
|
|
11
|
+
&& state.version === 1 && typeof state.done === "boolean"
|
|
12
|
+
&& [state.pending, state.visited].every(items => Array.isArray(items) && items.every(hash))
|
|
13
|
+
&& (state.cursor === null || /^[a-f0-9]{64}\.json$/.test(state.cursor))
|
|
14
|
+
&& (!state.done || state.pending.length === 0);
|
|
15
|
+
|
|
16
|
+
// This is optional, checksummed bookkeeping, never primary authority. Atomic
|
|
17
|
+
// replacement means process death leaves the previous complete checkpoint;
|
|
18
|
+
// lost/torn bytes restart marking before any further deletion. No fsync is due.
|
|
19
|
+
export async function readReclaimProgress(root) {
|
|
20
|
+
try {
|
|
21
|
+
const bytes = await withRegularNoFollow(file(root), root, "r", (handle, stat) =>
|
|
22
|
+
stat.size > 64 * 1024 * 1024 ? null : handle.readFile());
|
|
23
|
+
if (bytes === null) return fresh();
|
|
24
|
+
const value = JSON.parse(bytes.toString("utf8"));
|
|
25
|
+
if (!valid(value.state) || value.sha256 !== digest(JSON.stringify(value.state))) return fresh();
|
|
26
|
+
return value.state.done ? fresh() : value.state;
|
|
27
|
+
} catch { return fresh(); }
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export async function writeReclaimProgress(paths, root, state, deadlineAt) {
|
|
31
|
+
const body = JSON.stringify(state);
|
|
32
|
+
await publishAtomic(file(root), Buffer.from(JSON.stringify({ state, sha256: digest(body) }) + "\n"), {
|
|
33
|
+
root, tmpDir: paths.tmp, stageDir: paths.stage, replace: true, durability: "none", deadlineAt,
|
|
34
|
+
});
|
|
35
|
+
}
|