@xfey/tutti 0.1.107 → 0.1.108
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/dist/collaboration-ingestion/external-source.js +10 -1
- package/dist/control-plane/task-source-context-types.d.ts +2 -1
- package/dist/control-plane/task-source-context.js +7 -0
- package/dist/providers/openai/app-server/read-only-procedure.js +3 -0
- package/dist/providers/openai/app-server/skills.d.ts +4 -1
- package/dist/providers/openai/app-server/skills.js +12 -2
- package/dist/providers/openai/app-server/workspace-write-run.js +14 -10
- package/dist/reference-access/cache.d.ts +50 -0
- package/dist/reference-access/cache.js +268 -0
- package/dist/reference-access/read-evidence.d.ts +6 -0
- package/dist/reference-access/read-evidence.js +15 -0
- package/dist/reference-access/service.d.ts +26 -0
- package/dist/reference-access/service.js +42 -0
- package/dist/server-shell/cli/external-reference-runtime.d.ts +12 -0
- package/dist/server-shell/cli/external-reference-runtime.js +75 -0
- package/dist/server-shell/cli/host-server-runtime.js +17 -1
- package/dist/server-shell/cli/relay-registration.js +1 -0
- package/dist/server-shell/http/create-server.d.ts +2 -0
- package/dist/server-shell/http/create-server.js +5 -4
- package/dist/server-shell/http/routes/agent-context-auth.d.ts +2 -1
- package/dist/server-shell/http/routes/agent-context-auth.js +3 -1
- package/dist/server-shell/http/routes/agent-context-external-reference-routes.d.ts +4 -0
- package/dist/server-shell/http/routes/agent-context-external-reference-routes.js +83 -0
- package/dist/server-shell/http/routes/agent-context-shared.d.ts +2 -0
- package/dist/server-shell/http/routes/agent-context.js +3 -1
- package/dist/workspace-ops/project-workspace.js +6 -6
- package/dist/workspace-ops/reference-cache-policy.d.ts +6 -0
- package/dist/workspace-ops/reference-cache-policy.js +79 -0
- package/migrations/0021_external_reference_reads.sql +11 -0
- package/node_modules/@tutti/relay-client/dist/external-references.d.ts +11 -0
- package/node_modules/@tutti/relay-client/dist/external-references.js +63 -0
- package/node_modules/@tutti/relay-client/dist/host-control.d.ts +1 -0
- package/node_modules/@tutti/relay-client/dist/host-control.js +1 -0
- package/node_modules/@tutti/relay-client/dist/index.d.ts +1 -0
- package/node_modules/@tutti/relay-client/dist/index.js +1 -0
- package/node_modules/@tutti/shared/dist/schemas/api/clarifications.d.ts +6 -0
- package/node_modules/@tutti/shared/dist/schemas/api/external-references.d.ts +23 -0
- package/node_modules/@tutti/shared/dist/schemas/api/external-references.js +21 -0
- package/node_modules/@tutti/shared/dist/schemas/api/feishu-documents.d.ts +6 -1
- package/node_modules/@tutti/shared/dist/schemas/api/feishu-documents.js +16 -3
- package/node_modules/@tutti/shared/dist/schemas/api/index.d.ts +3 -2
- package/node_modules/@tutti/shared/dist/schemas/api/index.js +2 -1
- package/node_modules/@tutti/shared/dist/schemas/api/messages.d.ts +24 -0
- package/node_modules/@tutti/shared/dist/schemas/api/messages.js +2 -0
- package/node_modules/@tutti/shared/dist/schemas/api/platform-sessions.d.ts +7 -1
- package/node_modules/@tutti/shared/dist/schemas/api/platform-sessions.js +17 -3
- package/node_modules/@tutti/shared/dist/schemas/api/viewer-reference.d.ts +6 -0
- package/node_modules/@tutti/shared/dist/schemas/internal/external-references.d.ts +109 -0
- package/node_modules/@tutti/shared/dist/schemas/internal/external-references.js +70 -0
- package/node_modules/@tutti/shared/dist/schemas/internal/feishu-documents.d.ts +11 -0
- package/node_modules/@tutti/shared/dist/schemas/internal/feishu-documents.js +5 -0
- package/node_modules/@tutti/shared/dist/schemas/internal/index.d.ts +1 -0
- package/node_modules/@tutti/shared/dist/schemas/internal/index.js +1 -0
- package/node_modules/@tutti/shared/dist/schemas/internal/platform-integration.d.ts +36 -0
- package/node_modules/@tutti/shared/dist/schemas/internal/platform-integration.js +3 -0
- package/package.json +1 -1
- package/prompts/skills/read-references/SKILL.md +32 -3
- package/web/assets/{homepage-motion-scene-CFs5rMWq.js → homepage-motion-scene-B6yPJnY8.js} +1 -1
- package/web/assets/{index-BmiLZmaV.js → index-aQnJ5JoI.js} +3 -3
- package/web/index.html +1 -1
package/README.md
CHANGED
|
@@ -15,6 +15,7 @@
|
|
|
15
15
|
- host 到 Relay 的连接与工作区加入协作
|
|
16
16
|
- machine-local `standalone | feishu` Project Surface authority,以及Standalone browser session / Feishu trusted integration的双向Host gate
|
|
17
17
|
- Feishu主群聊Chat Assistant canonical response与Host-owned durable outbound handoff intent
|
|
18
|
+
- Feishu 外部来源的按需读取、受控临时 cache、invocation 版本和 SQLite 读取证据;正式 Reference/Standalone 上传与历史 snapshot 保持,见 [external-reference-access.md](../../docs/system/external-reference-access.md)
|
|
18
19
|
|
|
19
20
|
## 结构
|
|
20
21
|
|
|
@@ -86,7 +86,13 @@ export function ingestTrustedConversationSource(options) {
|
|
|
86
86
|
account_ref: options.command.actor.account_ref,
|
|
87
87
|
display_name: options.command.actor.display_name,
|
|
88
88
|
},
|
|
89
|
-
materialize: () => ({
|
|
89
|
+
materialize: () => ({
|
|
90
|
+
kind: "message",
|
|
91
|
+
body: source.content.text,
|
|
92
|
+
...(source.external_references === undefined
|
|
93
|
+
? {}
|
|
94
|
+
: { refs: { external_references: source.external_references } }),
|
|
95
|
+
}),
|
|
90
96
|
idempotencyContext: {
|
|
91
97
|
kind: "external_conversation_source",
|
|
92
98
|
platform: source.locator.conversation.installation.platform,
|
|
@@ -94,6 +100,9 @@ export function ingestTrustedConversationSource(options) {
|
|
|
94
100
|
external_message_ref: source.locator.message_ref,
|
|
95
101
|
binding_ref: options.command.binding_ref,
|
|
96
102
|
receipt_key: options.command.receipt.receipt_key,
|
|
103
|
+
...(source.external_references === undefined
|
|
104
|
+
? {}
|
|
105
|
+
: { external_references: source.external_references }),
|
|
97
106
|
},
|
|
98
107
|
afterCreate: (tx, message) => insertExternalSourceEvidence(tx, options.command, message),
|
|
99
108
|
...(options.scratchpadSource === undefined
|
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
import type { ClarificationRequestPayload } from "@tutti/shared/schemas/api";
|
|
1
|
+
import type { ClarificationRequestPayload, ExternalReferenceDescriptor } from "@tutti/shared/schemas/api";
|
|
2
2
|
export type PromptSourceMessage = {
|
|
3
3
|
role: "human" | "tutti";
|
|
4
4
|
author_display_name?: string;
|
|
5
5
|
text: string;
|
|
6
|
+
external_references?: ExternalReferenceDescriptor[];
|
|
6
7
|
};
|
|
7
8
|
export type TaskCompileClarificationContext = {
|
|
8
9
|
request_payload: ClarificationRequestPayload;
|
|
@@ -16,6 +16,13 @@ function promptSourceMessage(db, source) {
|
|
|
16
16
|
? {}
|
|
17
17
|
: { author_display_name: displayName }),
|
|
18
18
|
text: messageTextForModel(message),
|
|
19
|
+
...(message.refs?.external_references === undefined
|
|
20
|
+
? {}
|
|
21
|
+
: {
|
|
22
|
+
external_references: message.refs.external_references.map((reference) => ({
|
|
23
|
+
...reference,
|
|
24
|
+
})),
|
|
25
|
+
}),
|
|
19
26
|
};
|
|
20
27
|
}
|
|
21
28
|
if (message.message_kind === "agent_text" && message.author.kind === "agent") {
|
|
@@ -145,6 +145,9 @@ export async function runCodexAppServerReadOnlyProcedure(options) {
|
|
|
145
145
|
: { builtInSkillsRoot: options.builtInSkillsRoot }),
|
|
146
146
|
});
|
|
147
147
|
contextRuntimeHint = resolveCodexAppServerContextRuntimeHint({
|
|
148
|
+
...(options.runtimeTelemetry?.activityRef === undefined
|
|
149
|
+
? {}
|
|
150
|
+
: { activityRef: options.runtimeTelemetry.activityRef }),
|
|
148
151
|
...(options.skillSelectionName === undefined
|
|
149
152
|
? {}
|
|
150
153
|
: { selectionName: options.skillSelectionName }),
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { ActivityRef } from "@tutti/shared/ids";
|
|
1
2
|
export type CodexAppServerSkillInput = {
|
|
2
3
|
name: string;
|
|
3
4
|
path: string;
|
|
@@ -15,13 +16,14 @@ export type CodexAppServerSkillInputItem = {
|
|
|
15
16
|
export type CodexAppServerTurnInputItem = CodexAppServerSkillInputItem | CodexAppServerTextInputItem;
|
|
16
17
|
export type CodexAppServerBuiltInSkillName = "read-main-chat" | "read-project-docs" | "read-scratchpad" | "read-worklist" | "read-references" | "read-user-skills";
|
|
17
18
|
export type CodexAppServerSkillSelectionName = "chat_assistant" | "project_context_bootstrap" | "task_compile" | "context_sync" | "reference_summary_refresh" | "workspace_write_run" | "follow_up_run";
|
|
18
|
-
export type CodexAppServerContextApiScope = "messages:read" | "project-docs:list" | "project-docs:read" | "scratchpad:read" | "worklist:read" | "task:read" | "references:list" | "references:read" | "skills:list" | "skills:read";
|
|
19
|
+
export type CodexAppServerContextApiScope = "messages:read" | "project-docs:list" | "project-docs:read" | "scratchpad:read" | "worklist:read" | "task:read" | "references:list" | "references:read" | "references:external" | "skills:list" | "skills:read";
|
|
19
20
|
export type CodexAppServerSkillSelection = {
|
|
20
21
|
builtInSkillNames: CodexAppServerBuiltInSkillName[];
|
|
21
22
|
userSkillMode: "disabled" | "all_enabled";
|
|
22
23
|
contextApiScopes: CodexAppServerContextApiScope[];
|
|
23
24
|
};
|
|
24
25
|
export type CodexAppServerAgentContextTokenRequest = {
|
|
26
|
+
activityRef?: ActivityRef;
|
|
25
27
|
selectionName: CodexAppServerSkillSelectionName;
|
|
26
28
|
scopes: CodexAppServerContextApiScope[];
|
|
27
29
|
};
|
|
@@ -76,6 +78,7 @@ export declare function buildCodexAppServerContextRuntimeHint(options: {
|
|
|
76
78
|
expiresAt?: string;
|
|
77
79
|
}): string;
|
|
78
80
|
export declare function resolveCodexAppServerContextRuntimeHint(options: {
|
|
81
|
+
activityRef?: ActivityRef;
|
|
79
82
|
selectionName?: CodexAppServerSkillSelectionName;
|
|
80
83
|
agentContext?: CodexAppServerAgentContextRuntime;
|
|
81
84
|
}): CodexAppServerContextRuntimeHint | undefined;
|
|
@@ -30,6 +30,7 @@ export const CODEX_APP_SERVER_SKILL_SELECTIONS = {
|
|
|
30
30
|
"task:read",
|
|
31
31
|
"references:list",
|
|
32
32
|
"references:read",
|
|
33
|
+
"references:external",
|
|
33
34
|
"skills:list",
|
|
34
35
|
"skills:read",
|
|
35
36
|
],
|
|
@@ -42,6 +43,7 @@ export const CODEX_APP_SERVER_SKILL_SELECTIONS = {
|
|
|
42
43
|
"project-docs:read",
|
|
43
44
|
"references:list",
|
|
44
45
|
"references:read",
|
|
46
|
+
"references:external",
|
|
45
47
|
],
|
|
46
48
|
},
|
|
47
49
|
task_compile: {
|
|
@@ -54,6 +56,7 @@ export const CODEX_APP_SERVER_SKILL_SELECTIONS = {
|
|
|
54
56
|
"task:read",
|
|
55
57
|
"references:list",
|
|
56
58
|
"references:read",
|
|
59
|
+
"references:external",
|
|
57
60
|
],
|
|
58
61
|
},
|
|
59
62
|
context_sync: {
|
|
@@ -74,23 +77,29 @@ export const CODEX_APP_SERVER_SKILL_SELECTIONS = {
|
|
|
74
77
|
contextApiScopes: [],
|
|
75
78
|
},
|
|
76
79
|
workspace_write_run: {
|
|
77
|
-
builtInSkillNames: ["read-project-docs", "read-worklist"],
|
|
80
|
+
builtInSkillNames: ["read-project-docs", "read-worklist", "read-references"],
|
|
78
81
|
userSkillMode: "all_enabled",
|
|
79
82
|
contextApiScopes: [
|
|
80
83
|
"project-docs:list",
|
|
81
84
|
"project-docs:read",
|
|
82
85
|
"worklist:read",
|
|
83
86
|
"task:read",
|
|
87
|
+
"references:list",
|
|
88
|
+
"references:read",
|
|
89
|
+
"references:external",
|
|
84
90
|
],
|
|
85
91
|
},
|
|
86
92
|
follow_up_run: {
|
|
87
|
-
builtInSkillNames: ["read-project-docs", "read-worklist"],
|
|
93
|
+
builtInSkillNames: ["read-project-docs", "read-worklist", "read-references"],
|
|
88
94
|
userSkillMode: "all_enabled",
|
|
89
95
|
contextApiScopes: [
|
|
90
96
|
"project-docs:list",
|
|
91
97
|
"project-docs:read",
|
|
92
98
|
"worklist:read",
|
|
93
99
|
"task:read",
|
|
100
|
+
"references:list",
|
|
101
|
+
"references:read",
|
|
102
|
+
"references:external",
|
|
94
103
|
],
|
|
95
104
|
},
|
|
96
105
|
};
|
|
@@ -206,6 +215,7 @@ export function resolveCodexAppServerContextRuntimeHint(options) {
|
|
|
206
215
|
return undefined;
|
|
207
216
|
}
|
|
208
217
|
const issued = options.agentContext.issueToken({
|
|
218
|
+
...(options.activityRef === undefined ? {} : { activityRef: options.activityRef }),
|
|
209
219
|
selectionName: options.selectionName,
|
|
210
220
|
scopes: selection.contextApiScopes,
|
|
211
221
|
});
|
|
@@ -174,7 +174,9 @@ export async function runCodexAppServerWorkspaceWriteRun(options) {
|
|
|
174
174
|
let contextRuntimeHint;
|
|
175
175
|
try {
|
|
176
176
|
skillInputs = resolveCodexAppServerSkillInputs({
|
|
177
|
-
...(options.skillSelectionName === undefined
|
|
177
|
+
...(options.skillSelectionName === undefined
|
|
178
|
+
? {}
|
|
179
|
+
: { selectionName: options.skillSelectionName }),
|
|
178
180
|
...(options.userSkillsRoot === undefined ? {} : { userSkillsRoot: options.userSkillsRoot }),
|
|
179
181
|
...(options.builtInSkillsRoot === undefined
|
|
180
182
|
? {}
|
|
@@ -182,11 +184,18 @@ export async function runCodexAppServerWorkspaceWriteRun(options) {
|
|
|
182
184
|
});
|
|
183
185
|
skillsListWarmup = resolveCodexAppServerSkillsListWarmup({
|
|
184
186
|
cwd: options.workspaceRoot,
|
|
185
|
-
...(options.skillSelectionName === undefined
|
|
187
|
+
...(options.skillSelectionName === undefined
|
|
188
|
+
? {}
|
|
189
|
+
: { selectionName: options.skillSelectionName }),
|
|
186
190
|
...(options.userSkillsRoot === undefined ? {} : { userSkillsRoot: options.userSkillsRoot }),
|
|
187
191
|
});
|
|
188
192
|
contextRuntimeHint = resolveCodexAppServerContextRuntimeHint({
|
|
189
|
-
...(options.
|
|
193
|
+
...(options.approvalContext?.activityRef === undefined
|
|
194
|
+
? {}
|
|
195
|
+
: { activityRef: options.approvalContext.activityRef }),
|
|
196
|
+
...(options.skillSelectionName === undefined
|
|
197
|
+
? {}
|
|
198
|
+
: { selectionName: options.skillSelectionName }),
|
|
190
199
|
...(options.agentContext === undefined ? {} : { agentContext: options.agentContext }),
|
|
191
200
|
});
|
|
192
201
|
}
|
|
@@ -264,10 +273,7 @@ export async function runCodexAppServerWorkspaceWriteRun(options) {
|
|
|
264
273
|
...entry.extraUserRoots,
|
|
265
274
|
]) ?? []),
|
|
266
275
|
];
|
|
267
|
-
const strictLeakCheckValues = [
|
|
268
|
-
options.apiKey,
|
|
269
|
-
...(contextRuntimeHint?.sensitiveValues ?? []),
|
|
270
|
-
];
|
|
276
|
+
const strictLeakCheckValues = [options.apiKey, ...(contextRuntimeHint?.sensitiveValues ?? [])];
|
|
271
277
|
let primaryError;
|
|
272
278
|
let runResult;
|
|
273
279
|
let lifecycleError;
|
|
@@ -336,9 +342,7 @@ export async function runCodexAppServerWorkspaceWriteRun(options) {
|
|
|
336
342
|
input: buildCodexAppServerTurnInput({
|
|
337
343
|
prompt: options.prompt,
|
|
338
344
|
skills: skillInputs,
|
|
339
|
-
...(contextRuntimeHint === undefined
|
|
340
|
-
? {}
|
|
341
|
-
: { runtimeHints: [contextRuntimeHint.text] }),
|
|
345
|
+
...(contextRuntimeHint === undefined ? {} : { runtimeHints: [contextRuntimeHint.text] }),
|
|
342
346
|
}),
|
|
343
347
|
cwd: options.workspaceRoot,
|
|
344
348
|
approvalPolicy,
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
export declare class ReferenceCacheError extends Error {
|
|
2
|
+
readonly code: string;
|
|
3
|
+
constructor(code: string);
|
|
4
|
+
}
|
|
5
|
+
export type ReferenceCopy = {
|
|
6
|
+
source_ref: string;
|
|
7
|
+
revision: string;
|
|
8
|
+
sha256: string;
|
|
9
|
+
path: string;
|
|
10
|
+
size_bytes: number;
|
|
11
|
+
media_type: string;
|
|
12
|
+
checked_at: string;
|
|
13
|
+
};
|
|
14
|
+
export type ReferenceFetchResult = {
|
|
15
|
+
revision: string;
|
|
16
|
+
media_type: string;
|
|
17
|
+
content: Uint8Array;
|
|
18
|
+
};
|
|
19
|
+
export declare class ExternalReferenceCache {
|
|
20
|
+
private readonly options;
|
|
21
|
+
private readonly workspace;
|
|
22
|
+
private readonly ledgerFile;
|
|
23
|
+
private readonly ledger;
|
|
24
|
+
private tail;
|
|
25
|
+
constructor(options: {
|
|
26
|
+
workspaceRoot: string;
|
|
27
|
+
ledgerRoot: string;
|
|
28
|
+
now?: () => number;
|
|
29
|
+
maxTotalBytes?: number;
|
|
30
|
+
retentionMs?: number;
|
|
31
|
+
});
|
|
32
|
+
private now;
|
|
33
|
+
private validateLedger;
|
|
34
|
+
private save;
|
|
35
|
+
private bytes;
|
|
36
|
+
/** A pin belongs to a server-issued invocation, never to model-supplied identity. */
|
|
37
|
+
acquire(input: {
|
|
38
|
+
sourceRef: string;
|
|
39
|
+
invocationRef: string;
|
|
40
|
+
expiresAt: number;
|
|
41
|
+
fetch: () => Promise<ReferenceFetchResult>;
|
|
42
|
+
}): Promise<{
|
|
43
|
+
copy: ReferenceCopy;
|
|
44
|
+
cache_hit: boolean;
|
|
45
|
+
content: Buffer;
|
|
46
|
+
}>;
|
|
47
|
+
private projection;
|
|
48
|
+
private prune;
|
|
49
|
+
}
|
|
50
|
+
//# sourceMappingURL=cache.d.ts.map
|
|
@@ -0,0 +1,268 @@
|
|
|
1
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
2
|
+
import { lstatSync, mkdirSync, readFileSync, realpathSync, renameSync, unlinkSync, writeFileSync, } from "node:fs";
|
|
3
|
+
import { dirname, join, resolve } from "node:path";
|
|
4
|
+
import { assertReferenceCacheIsolation, initializeReferenceCacheDirectory, REFERENCE_CACHE_PATH, } from "../workspace-ops/reference-cache-policy.js";
|
|
5
|
+
const MAX_BYTES = 10 * 1024 * 1024;
|
|
6
|
+
const SAFE_REF = /^[A-Za-z0-9_-]{1,128}$/u;
|
|
7
|
+
const DIGEST = /^[a-f0-9]{64}$/u;
|
|
8
|
+
const MAX_PIN_MS = 24 * 60 * 60 * 1000;
|
|
9
|
+
export class ReferenceCacheError extends Error {
|
|
10
|
+
code;
|
|
11
|
+
constructor(code) {
|
|
12
|
+
super(code);
|
|
13
|
+
this.code = code;
|
|
14
|
+
this.name = "ReferenceCacheError";
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
function fail() {
|
|
18
|
+
throw new ReferenceCacheError("reference_cache_invalid");
|
|
19
|
+
}
|
|
20
|
+
function hash(bytes) {
|
|
21
|
+
return createHash("sha256").update(bytes).digest("hex");
|
|
22
|
+
}
|
|
23
|
+
function regular(path) {
|
|
24
|
+
const stat = lstatSync(path, { throwIfNoEntry: false });
|
|
25
|
+
if (stat === undefined)
|
|
26
|
+
return false;
|
|
27
|
+
if (!stat.isFile() || stat.isSymbolicLink() || stat.nlink !== 1)
|
|
28
|
+
fail();
|
|
29
|
+
return true;
|
|
30
|
+
}
|
|
31
|
+
function ensureDirectories(path) {
|
|
32
|
+
const absolute = resolve(path);
|
|
33
|
+
const parent = dirname(absolute);
|
|
34
|
+
if (parent !== absolute)
|
|
35
|
+
ensureDirectories(parent);
|
|
36
|
+
const stat = lstatSync(absolute, { throwIfNoEntry: false });
|
|
37
|
+
if (stat !== undefined) {
|
|
38
|
+
if (!stat.isDirectory() || stat.isSymbolicLink())
|
|
39
|
+
fail();
|
|
40
|
+
}
|
|
41
|
+
else
|
|
42
|
+
mkdirSync(absolute, { mode: 0o700 });
|
|
43
|
+
}
|
|
44
|
+
function relativeCopyPath(source, digest) {
|
|
45
|
+
return `${REFERENCE_CACHE_PATH}/${source}/${digest}.bin`;
|
|
46
|
+
}
|
|
47
|
+
export class ExternalReferenceCache {
|
|
48
|
+
options;
|
|
49
|
+
workspace;
|
|
50
|
+
ledgerFile;
|
|
51
|
+
ledger;
|
|
52
|
+
tail = Promise.resolve();
|
|
53
|
+
constructor(options) {
|
|
54
|
+
this.options = options;
|
|
55
|
+
this.workspace = realpathSync(options.workspaceRoot);
|
|
56
|
+
if ((options.maxTotalBytes !== undefined &&
|
|
57
|
+
(!Number.isSafeInteger(options.maxTotalBytes) || options.maxTotalBytes < 1)) ||
|
|
58
|
+
(options.retentionMs !== undefined &&
|
|
59
|
+
(!Number.isSafeInteger(options.retentionMs) || options.retentionMs < 0)))
|
|
60
|
+
fail();
|
|
61
|
+
const ledgerRoot = resolve(options.ledgerRoot);
|
|
62
|
+
if (ledgerRoot === this.workspace || ledgerRoot.startsWith(`${this.workspace}/`))
|
|
63
|
+
fail();
|
|
64
|
+
ensureDirectories(ledgerRoot);
|
|
65
|
+
this.ledgerFile = join(ledgerRoot, "cache.json");
|
|
66
|
+
if (regular(this.ledgerFile)) {
|
|
67
|
+
try {
|
|
68
|
+
const parsed = JSON.parse(readFileSync(this.ledgerFile, "utf8"));
|
|
69
|
+
this.ledger = this.validateLedger(parsed);
|
|
70
|
+
}
|
|
71
|
+
catch {
|
|
72
|
+
fail();
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
else {
|
|
76
|
+
this.ledger = { version: 1, workspace: this.workspace, entries: {}, pins: {} };
|
|
77
|
+
}
|
|
78
|
+
initializeReferenceCacheDirectory(this.workspace);
|
|
79
|
+
this.save();
|
|
80
|
+
}
|
|
81
|
+
now() {
|
|
82
|
+
return (this.options.now ?? Date.now)();
|
|
83
|
+
}
|
|
84
|
+
validateLedger(value) {
|
|
85
|
+
if (value === null || typeof value !== "object")
|
|
86
|
+
fail();
|
|
87
|
+
const candidate = value;
|
|
88
|
+
if (candidate.version !== 1 ||
|
|
89
|
+
candidate.workspace !== this.workspace ||
|
|
90
|
+
!candidate.entries ||
|
|
91
|
+
!candidate.pins ||
|
|
92
|
+
typeof candidate.entries !== "object" ||
|
|
93
|
+
typeof candidate.pins !== "object" ||
|
|
94
|
+
Array.isArray(candidate.entries) ||
|
|
95
|
+
Array.isArray(candidate.pins))
|
|
96
|
+
fail();
|
|
97
|
+
for (const [key, entry] of Object.entries(candidate.entries)) {
|
|
98
|
+
if (!entry ||
|
|
99
|
+
!SAFE_REF.test(entry.source_ref) ||
|
|
100
|
+
!DIGEST.test(entry.sha256) ||
|
|
101
|
+
key !== `${entry.source_ref}:${entry.sha256}` ||
|
|
102
|
+
entry.path !== relativeCopyPath(entry.source_ref, entry.sha256) ||
|
|
103
|
+
typeof entry.revision !== "string" ||
|
|
104
|
+
entry.revision.length > 256 ||
|
|
105
|
+
typeof entry.media_type !== "string" ||
|
|
106
|
+
entry.media_type.length > 128 ||
|
|
107
|
+
!Number.isSafeInteger(entry.size_bytes) ||
|
|
108
|
+
entry.size_bytes < 0 ||
|
|
109
|
+
entry.size_bytes > MAX_BYTES ||
|
|
110
|
+
!Number.isFinite(entry.last_used_at) ||
|
|
111
|
+
!Number.isFinite(Date.parse(entry.checked_at)))
|
|
112
|
+
fail();
|
|
113
|
+
}
|
|
114
|
+
for (const [key, pin] of Object.entries(candidate.pins)) {
|
|
115
|
+
if (!/^[a-f0-9]{64}$/u.test(key) ||
|
|
116
|
+
!pin ||
|
|
117
|
+
!Number.isFinite(pin.expires_at) ||
|
|
118
|
+
!Object.hasOwn(candidate.entries, pin.key) ||
|
|
119
|
+
!pin.copy ||
|
|
120
|
+
pin.copy.path !== candidate.entries[pin.key]?.path ||
|
|
121
|
+
pin.copy.sha256 !== candidate.entries[pin.key]?.sha256 ||
|
|
122
|
+
pin.copy.source_ref !== candidate.entries[pin.key]?.source_ref ||
|
|
123
|
+
pin.copy.size_bytes !== candidate.entries[pin.key]?.size_bytes ||
|
|
124
|
+
typeof pin.copy.media_type !== "string" ||
|
|
125
|
+
pin.copy.media_type.length > 128 ||
|
|
126
|
+
typeof pin.copy.revision !== "string" ||
|
|
127
|
+
!pin.copy.revision ||
|
|
128
|
+
pin.copy.revision.length > 256 ||
|
|
129
|
+
!Number.isFinite(Date.parse(pin.copy.checked_at)))
|
|
130
|
+
fail();
|
|
131
|
+
}
|
|
132
|
+
return candidate;
|
|
133
|
+
}
|
|
134
|
+
save() {
|
|
135
|
+
regular(this.ledgerFile);
|
|
136
|
+
const temporary = `${this.ledgerFile}.${randomUUID()}.tmp`;
|
|
137
|
+
writeFileSync(temporary, JSON.stringify(this.ledger), { flag: "wx", mode: 0o600 });
|
|
138
|
+
renameSync(temporary, this.ledgerFile);
|
|
139
|
+
}
|
|
140
|
+
bytes(entry) {
|
|
141
|
+
assertReferenceCacheIsolation(this.workspace, true);
|
|
142
|
+
const directory = join(this.workspace, REFERENCE_CACHE_PATH, entry.source_ref);
|
|
143
|
+
const stat = lstatSync(directory, { throwIfNoEntry: false });
|
|
144
|
+
if (!stat?.isDirectory() || stat.isSymbolicLink())
|
|
145
|
+
fail();
|
|
146
|
+
const path = join(this.workspace, entry.path);
|
|
147
|
+
if (!regular(path))
|
|
148
|
+
throw new ReferenceCacheError("reference_cache_missing");
|
|
149
|
+
if (lstatSync(path).size !== entry.size_bytes)
|
|
150
|
+
fail();
|
|
151
|
+
const content = readFileSync(path);
|
|
152
|
+
if (hash(content) !== entry.sha256)
|
|
153
|
+
fail();
|
|
154
|
+
return content;
|
|
155
|
+
}
|
|
156
|
+
/** A pin belongs to a server-issued invocation, never to model-supplied identity. */
|
|
157
|
+
async acquire(input) {
|
|
158
|
+
if (!SAFE_REF.test(input.sourceRef) ||
|
|
159
|
+
!SAFE_REF.test(input.invocationRef) ||
|
|
160
|
+
!Number.isFinite(input.expiresAt) ||
|
|
161
|
+
input.expiresAt <= this.now() ||
|
|
162
|
+
input.expiresAt > this.now() + MAX_PIN_MS)
|
|
163
|
+
fail();
|
|
164
|
+
const operation = this.tail.then(async () => {
|
|
165
|
+
if (input.expiresAt <= this.now())
|
|
166
|
+
throw new ReferenceCacheError("reference_invocation_expired");
|
|
167
|
+
assertReferenceCacheIsolation(this.workspace, true);
|
|
168
|
+
const pinKey = hash(Buffer.from(`${input.invocationRef}\0${input.sourceRef}`));
|
|
169
|
+
const pin = this.ledger.pins[pinKey];
|
|
170
|
+
if (pin !== undefined && pin.expires_at > this.now()) {
|
|
171
|
+
const entry = this.ledger.entries[pin.key];
|
|
172
|
+
if (entry === undefined)
|
|
173
|
+
fail();
|
|
174
|
+
const content = this.bytes(entry);
|
|
175
|
+
entry.last_used_at = this.now();
|
|
176
|
+
this.save();
|
|
177
|
+
return { copy: pin.copy, cache_hit: true, content };
|
|
178
|
+
}
|
|
179
|
+
// A new invocation checks access and content. Failures never silently use stale bytes.
|
|
180
|
+
const fetched = await input.fetch();
|
|
181
|
+
assertReferenceCacheIsolation(this.workspace, true);
|
|
182
|
+
if (input.expiresAt <= this.now())
|
|
183
|
+
throw new ReferenceCacheError("reference_invocation_expired");
|
|
184
|
+
if (fetched.content.byteLength > MAX_BYTES ||
|
|
185
|
+
!fetched.revision ||
|
|
186
|
+
fetched.revision.length > 256 ||
|
|
187
|
+
!fetched.media_type ||
|
|
188
|
+
fetched.media_type.length > 128)
|
|
189
|
+
fail();
|
|
190
|
+
const content = Buffer.from(fetched.content);
|
|
191
|
+
const digest = hash(content);
|
|
192
|
+
const key = `${input.sourceRef}:${digest}`;
|
|
193
|
+
const existing = this.ledger.entries[key];
|
|
194
|
+
this.prune(content.byteLength, key);
|
|
195
|
+
const entry = {
|
|
196
|
+
source_ref: input.sourceRef,
|
|
197
|
+
revision: fetched.revision,
|
|
198
|
+
sha256: digest,
|
|
199
|
+
path: relativeCopyPath(input.sourceRef, digest),
|
|
200
|
+
size_bytes: content.byteLength,
|
|
201
|
+
media_type: fetched.media_type,
|
|
202
|
+
checked_at: new Date(this.now()).toISOString(),
|
|
203
|
+
last_used_at: this.now(),
|
|
204
|
+
};
|
|
205
|
+
ensureDirectories(join(this.workspace, REFERENCE_CACHE_PATH, input.sourceRef));
|
|
206
|
+
const path = join(this.workspace, entry.path);
|
|
207
|
+
if (existing !== undefined && regular(path))
|
|
208
|
+
this.bytes(existing);
|
|
209
|
+
else if (regular(path)) {
|
|
210
|
+
// Response-loss/crash orphan: adopt only an exact content-addressed regular file.
|
|
211
|
+
if (!readFileSync(path).equals(content))
|
|
212
|
+
fail();
|
|
213
|
+
}
|
|
214
|
+
else {
|
|
215
|
+
const temporary = `${path}.${randomUUID()}.tmp`;
|
|
216
|
+
writeFileSync(temporary, content, { flag: "wx", mode: 0o600 });
|
|
217
|
+
renameSync(temporary, path);
|
|
218
|
+
}
|
|
219
|
+
// Each version records its own observation. Pins capture revision independently below.
|
|
220
|
+
this.ledger.entries[key] = entry;
|
|
221
|
+
this.ledger.pins[pinKey] = { key, expires_at: input.expiresAt, copy: this.projection(entry) };
|
|
222
|
+
this.save();
|
|
223
|
+
return { copy: this.projection(entry), cache_hit: existing !== undefined, content };
|
|
224
|
+
});
|
|
225
|
+
this.tail = operation.catch(() => undefined);
|
|
226
|
+
return operation;
|
|
227
|
+
}
|
|
228
|
+
projection(entry) {
|
|
229
|
+
return {
|
|
230
|
+
source_ref: entry.source_ref,
|
|
231
|
+
revision: entry.revision,
|
|
232
|
+
sha256: entry.sha256,
|
|
233
|
+
path: entry.path,
|
|
234
|
+
size_bytes: entry.size_bytes,
|
|
235
|
+
media_type: entry.media_type,
|
|
236
|
+
checked_at: entry.checked_at,
|
|
237
|
+
};
|
|
238
|
+
}
|
|
239
|
+
prune(incomingBytes, incomingKey) {
|
|
240
|
+
const now = this.now();
|
|
241
|
+
for (const [key, pin] of Object.entries(this.ledger.pins)) {
|
|
242
|
+
if (pin.expires_at <= now)
|
|
243
|
+
delete this.ledger.pins[key];
|
|
244
|
+
}
|
|
245
|
+
const protectedKeys = new Set(Object.values(this.ledger.pins).map((pin) => pin.key));
|
|
246
|
+
const entries = Object.entries(this.ledger.entries).sort((a, b) => a[1].last_used_at - b[1].last_used_at);
|
|
247
|
+
let total = entries.reduce((sum, [, entry]) => sum + entry.size_bytes, 0);
|
|
248
|
+
const additional = Object.hasOwn(this.ledger.entries, incomingKey) ? 0 : incomingBytes;
|
|
249
|
+
const maximum = this.options.maxTotalBytes ?? 100 * 1024 * 1024;
|
|
250
|
+
const retention = this.options.retentionMs ?? 7 * 24 * 60 * 60 * 1000;
|
|
251
|
+
for (const [key, entry] of entries) {
|
|
252
|
+
if (key === incomingKey || protectedKeys.has(key))
|
|
253
|
+
continue;
|
|
254
|
+
if (total + additional <= maximum && entry.last_used_at + retention > now)
|
|
255
|
+
continue;
|
|
256
|
+
if (regular(join(this.workspace, entry.path))) {
|
|
257
|
+
this.bytes(entry); // Reject replacement/symlink/tampering before removal.
|
|
258
|
+
unlinkSync(join(this.workspace, entry.path));
|
|
259
|
+
}
|
|
260
|
+
delete this.ledger.entries[key];
|
|
261
|
+
total -= entry.size_bytes;
|
|
262
|
+
}
|
|
263
|
+
this.save();
|
|
264
|
+
if (total + additional > maximum)
|
|
265
|
+
throw new ReferenceCacheError("reference_cache_capacity");
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
//# sourceMappingURL=cache.js.map
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { SqliteDatabase } from "../store/index.js";
|
|
2
|
+
import type { ReferenceCopy } from "./cache.js";
|
|
3
|
+
import type { ReferenceInvocation } from "./service.js";
|
|
4
|
+
/** Durable observation, separate from evictable cache metadata; no content or locator. */
|
|
5
|
+
export declare function recordExternalReferenceRead(db: SqliteDatabase, invocation: ReferenceInvocation, copy: ReferenceCopy): void;
|
|
6
|
+
//# sourceMappingURL=read-evidence.d.ts.map
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/** Durable observation, separate from evictable cache metadata; no content or locator. */
|
|
2
|
+
export function recordExternalReferenceRead(db, invocation, copy) {
|
|
3
|
+
const invocationRef = invocation.reference_invocation_ref;
|
|
4
|
+
const existing = db
|
|
5
|
+
.prepare(`
|
|
6
|
+
SELECT revision, content_sha256 FROM external_reference_reads WHERE invocation_ref = ? AND source_ref = ?`)
|
|
7
|
+
.get(invocationRef, copy.source_ref);
|
|
8
|
+
if (existing &&
|
|
9
|
+
(existing.revision !== copy.revision || existing.content_sha256 !== copy.sha256)) {
|
|
10
|
+
throw new Error("reference_read_evidence_conflict");
|
|
11
|
+
}
|
|
12
|
+
db.prepare(`INSERT OR IGNORE INTO external_reference_reads
|
|
13
|
+
(invocation_ref, activity_ref, source_ref, revision, content_sha256, size_bytes, checked_at) VALUES (?, ?, ?, ?, ?, ?, ?)`).run(invocationRef, invocation.activity_ref ?? null, copy.source_ref, copy.revision, copy.sha256, copy.size_bytes, copy.checked_at);
|
|
14
|
+
}
|
|
15
|
+
//# sourceMappingURL=read-evidence.js.map
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import type { ExternalReferenceDescriptor } from "@tutti/shared/schemas/api";
|
|
2
|
+
import type { ReferenceCopy, ReferenceFetchResult } from "./cache.js";
|
|
3
|
+
import { type ExternalReferenceCache } from "./cache.js";
|
|
4
|
+
export type ReferenceInvocation = {
|
|
5
|
+
reference_invocation_ref: string;
|
|
6
|
+
activity_ref?: string;
|
|
7
|
+
expires_at: string;
|
|
8
|
+
};
|
|
9
|
+
export declare class ExternalReferenceAccess {
|
|
10
|
+
private readonly options;
|
|
11
|
+
private cache;
|
|
12
|
+
private readonly selections;
|
|
13
|
+
constructor(options: {
|
|
14
|
+
createCache: () => ExternalReferenceCache;
|
|
15
|
+
list: (invocation: ReferenceInvocation) => Promise<ExternalReferenceDescriptor[]>;
|
|
16
|
+
read: (sourceRef: string, invocation: ReferenceInvocation) => Promise<ReferenceFetchResult>;
|
|
17
|
+
recordRead: (invocation: ReferenceInvocation, copy: ReferenceCopy) => void;
|
|
18
|
+
});
|
|
19
|
+
list(invocation: ReferenceInvocation): Promise<ExternalReferenceDescriptor[]>;
|
|
20
|
+
read(invocation: ReferenceInvocation, sourceRef: string): Promise<{
|
|
21
|
+
copy: ReferenceCopy;
|
|
22
|
+
cache_hit: boolean;
|
|
23
|
+
content: Buffer;
|
|
24
|
+
}>;
|
|
25
|
+
}
|
|
26
|
+
//# sourceMappingURL=service.d.ts.map
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import { ReferenceCacheError } from "./cache.js";
|
|
2
|
+
export class ExternalReferenceAccess {
|
|
3
|
+
options;
|
|
4
|
+
cache;
|
|
5
|
+
selections = new Map();
|
|
6
|
+
constructor(options) {
|
|
7
|
+
this.options = options;
|
|
8
|
+
}
|
|
9
|
+
async list(invocation) {
|
|
10
|
+
const now = Date.now();
|
|
11
|
+
for (const [key, selection] of this.selections)
|
|
12
|
+
if (selection.expires <= now)
|
|
13
|
+
this.selections.delete(key);
|
|
14
|
+
let selection = this.selections.get(invocation.reference_invocation_ref);
|
|
15
|
+
if (!selection) {
|
|
16
|
+
if (this.selections.size >= 128)
|
|
17
|
+
throw new ReferenceCacheError("reference_reader_busy");
|
|
18
|
+
selection = {
|
|
19
|
+
expires: Date.parse(invocation.expires_at),
|
|
20
|
+
sources: this.options.list(invocation),
|
|
21
|
+
};
|
|
22
|
+
this.selections.set(invocation.reference_invocation_ref, selection);
|
|
23
|
+
selection.sources.catch(() => this.selections.delete(invocation.reference_invocation_ref));
|
|
24
|
+
}
|
|
25
|
+
return selection.sources;
|
|
26
|
+
}
|
|
27
|
+
async read(invocation, sourceRef) {
|
|
28
|
+
const sources = await this.list(invocation);
|
|
29
|
+
if (!sources.some((source) => source.source_ref === sourceRef))
|
|
30
|
+
throw new ReferenceCacheError("reference_source_not_selected");
|
|
31
|
+
this.cache ??= this.options.createCache();
|
|
32
|
+
const result = await this.cache.acquire({
|
|
33
|
+
sourceRef,
|
|
34
|
+
invocationRef: invocation.reference_invocation_ref,
|
|
35
|
+
expiresAt: Date.parse(invocation.expires_at),
|
|
36
|
+
fetch: () => this.options.read(sourceRef, invocation),
|
|
37
|
+
});
|
|
38
|
+
this.options.recordRead(invocation, result.copy);
|
|
39
|
+
return result;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
//# sourceMappingURL=service.js.map
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { ExternalReferenceAccess } from "../../reference-access/service.js";
|
|
2
|
+
import type { HostProjectStore } from "../../store/index.js";
|
|
3
|
+
import { type StagedUploadResolver } from "../http/routes/project-api/staged-uploads.js";
|
|
4
|
+
import type { HostLocalLaunchStatus } from "../http/routes/local-control.js";
|
|
5
|
+
import type { LaunchPreparationResult } from "./launch.js";
|
|
6
|
+
export declare function createExternalReferenceRuntime(options: {
|
|
7
|
+
preparation: LaunchPreparationResult;
|
|
8
|
+
store: HostProjectStore;
|
|
9
|
+
resolveStagedUpload: StagedUploadResolver;
|
|
10
|
+
getLaunchStatus: () => HostLocalLaunchStatus;
|
|
11
|
+
}): ExternalReferenceAccess;
|
|
12
|
+
//# sourceMappingURL=external-reference-runtime.d.ts.map
|