@gitroll/tacitt-cli 0.6.2 → 0.6.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{agent-setup-Dv_mQ38P.mjs → agent-setup-C7iqLMYm.mjs} +2 -2
- package/dist/agent-setup-worker.mjs +5 -5
- package/dist/cli.mjs +9 -180
- package/dist/codex-title-worker.mjs +1 -1
- package/dist/configured-preview-worker.mjs +5 -5
- package/dist/configured-session-worker.mjs +5 -5
- package/dist/{frame-DGun31Uc.mjs → frame-BGFmADbg.mjs} +1 -1
- package/dist/{heap-BR8WjR25.mjs → heap-DQbsSUrB.mjs} +18 -55
- package/dist/inspection-worker.mjs +4 -4
- package/dist/job-D4N5Pe91.mjs +6284 -0
- package/dist/{locate-JYtFmKu-.mjs → locate-CNU4z9G1.mjs} +4 -0
- package/dist/{model-C1VzgKLi.mjs → model-CZeHzKif.mjs} +1 -1
- package/dist/operation-executor.mjs +3 -3
- package/dist/preview-worker.mjs +1 -1
- package/dist/privacy-validator.mjs +1 -1
- package/dist/{query-CQ6Tr5zC.mjs → query-BMvtisKu.mjs} +1413 -5502
- package/dist/{run-hQ_W2p_Y.mjs → run-u2EfHjz4.mjs} +4 -4
- package/dist/{src-JoT3L94j.mjs → src-BRwicVza.mjs} +6 -4
- package/dist/{src-CGFwLUd6.mjs → src-BgoSvnXe.mjs} +4 -4
- package/dist/{src--Ln5GHdz.mjs → src-DmSIVU7-.mjs} +130 -13
- package/dist/store-database-worker.mjs +1 -1
- package/dist/web/assets/index-CndETc8f.js +169 -0
- package/dist/web/index.html +1 -1
- package/package.json +1 -1
- package/skills/tacitt/INDEX.md +1 -1
- package/skills/tacitt/SKILL.md +4 -2
- package/skills/tacitt/references/INDEX.md +2 -2
- package/skills/tacitt/references/commands.md +2 -0
- package/skills/tacitt/references/journey.md +3 -1
- package/skills/tacitt/references/remote-workers.md +6 -0
- package/skills/tacitt/references/send.md +3 -2
- package/skills/tacitt/scripts/INDEX.md +36 -36
- package/skills/tacitt/scripts/answer.mjs +41 -11
- package/skills/tacitt/scripts/journey-card-consolidation.d.mts +2 -0
- package/skills/tacitt/scripts/journey-card-consolidation.mjs +80 -5
- package/skills/tacitt/scripts/journey-card-values.d.mts +4 -0
- package/skills/tacitt/scripts/journey-card-values.mjs +21 -7
- package/skills/tacitt/scripts/journey-tasks.d.mts +1 -0
- package/skills/tacitt/scripts/journey-tasks.mjs +28 -12
- package/skills/tacitt/scripts/journey.mjs +3 -2
- package/dist/job-CYsjL4AT.mjs +0 -990
- package/dist/web/assets/index-C9LcK9ls.js +0 -169
package/dist/job-CYsjL4AT.mjs
DELETED
|
@@ -1,990 +0,0 @@
|
|
|
1
|
-
import { A as renderJobDigest, D as prepareCollectionProduction, V as writeLogicalJourneyOutput, W as readConfiguredOutputInfo, at as windowsOwnerOnly, b as writeLogicalJourneyBase, d as verifyOutputShard, f as writeOutputShard, nt as makePrivateDirectory, w as readConfiguredSearchFields, z as gitSeedsProvider } from "./src-JoT3L94j.mjs";
|
|
2
|
-
import { Xt as ApplicationError, nn as reviewFamilies, zt as filterInventoryByPeriod } from "./locate-JYtFmKu-.mjs";
|
|
3
|
-
import { Ar as staticSessionResourceSchema, Cn as renderedHideValue, Dn as hideKeyId, En as addHideIndexTokens, di as defaultScopedSettings, fi as freezeConfiguration, kn as packHideIndex, pi as selectFrozenConfiguration } from "./src--Ln5GHdz.mjs";
|
|
4
|
-
import { constants } from "node:fs";
|
|
5
|
-
import { z } from "zod";
|
|
6
|
-
import { copyFile, link, lstat, mkdir, mkdtemp, open, readFile, realpath, rename, rm, unlink, writeFile } from "node:fs/promises";
|
|
7
|
-
import { execFile } from "node:child_process";
|
|
8
|
-
import { dirname, join } from "node:path";
|
|
9
|
-
import { promisify } from "node:util";
|
|
10
|
-
import { createHash, randomBytes, randomUUID } from "node:crypto";
|
|
11
|
-
import { tmpdir } from "node:os";
|
|
12
|
-
import { quote } from "shell-quote";
|
|
13
|
-
//#region apps/cli/src/remote/model.ts
|
|
14
|
-
function validRemotePath(path, harness) {
|
|
15
|
-
return harness === "codex" ? /^\.codex\/sessions\/\d{4}\/\d{2}\/\d{2}\/rollout-[A-Za-z0-9_.-]+\.jsonl$/u.test(path) : /^\.claude\/projects\/[A-Za-z0-9_-]+\/[A-Za-z0-9_-]+\.jsonl$/u.test(path);
|
|
16
|
-
}
|
|
17
|
-
function insideWindow(start, window) {
|
|
18
|
-
const time = Date.parse(start);
|
|
19
|
-
return Number.isFinite(time) && time >= Date.parse(window.start) && time <= Date.parse(window.end);
|
|
20
|
-
}
|
|
21
|
-
//#endregion
|
|
22
|
-
//#region apps/cli/src/remote/transport.ts
|
|
23
|
-
const execute = promisify(execFile);
|
|
24
|
-
/** Pin every component to the signed-in user's home without following links. */
|
|
25
|
-
const sourceReader = `def source_open(relative):
|
|
26
|
-
parts=relative.split('/')
|
|
27
|
-
if any(part in ('','.', '..') for part in parts): raise RuntimeError('invalid source')
|
|
28
|
-
directory=os.open(os.path.expanduser('~'),os.O_RDONLY|os.O_DIRECTORY|os.O_NOFOLLOW)
|
|
29
|
-
try:
|
|
30
|
-
for part in parts[:-1]:
|
|
31
|
-
child=os.open(part,os.O_RDONLY|os.O_DIRECTORY|os.O_NOFOLLOW,dir_fd=directory)
|
|
32
|
-
os.close(directory)
|
|
33
|
-
directory=child
|
|
34
|
-
descriptor=os.open(parts[-1],os.O_RDONLY|os.O_NOFOLLOW|os.O_NONBLOCK,dir_fd=directory)
|
|
35
|
-
try:
|
|
36
|
-
metadata=os.fstat(descriptor)
|
|
37
|
-
if not stat.S_ISREG(metadata.st_mode) or metadata.st_uid!=os.getuid(): raise RuntimeError('invalid source owner or type')
|
|
38
|
-
return os.fdopen(descriptor,'rb')
|
|
39
|
-
except BaseException:
|
|
40
|
-
os.close(descriptor)
|
|
41
|
-
raise
|
|
42
|
-
finally: os.close(directory)`;
|
|
43
|
-
/** Read only a bounded prefix, ignoring Claude bookkeeping before native session records. */
|
|
44
|
-
const startReader = `def session_start(f,harness):
|
|
45
|
-
prefix=f.read(262145)
|
|
46
|
-
lines=prefix[:262144].split(b'\\n')
|
|
47
|
-
if len(prefix)>262144: lines.pop()
|
|
48
|
-
types=('user','assistant','system','progress','summary','result','queue-operation')
|
|
49
|
-
for line in lines:
|
|
50
|
-
if not line.strip(): continue
|
|
51
|
-
record=json.loads(line)
|
|
52
|
-
if not isinstance(record,dict): continue
|
|
53
|
-
if harness=='codex' and record.get('type')!='session_meta': return None
|
|
54
|
-
if harness=='claude' and record.get('type') not in types: continue
|
|
55
|
-
time=record.get('timestamp')
|
|
56
|
-
if isinstance(time,str):
|
|
57
|
-
try:
|
|
58
|
-
stamp(time)
|
|
59
|
-
return time
|
|
60
|
-
except ValueError: pass
|
|
61
|
-
if harness=='codex': return None
|
|
62
|
-
return None`;
|
|
63
|
-
/** Fixed remote program: bounded first-record metadata only, never directory copying. */
|
|
64
|
-
const listing = `import os,glob,json,datetime,stat
|
|
65
|
-
harness,start,end=INPUT
|
|
66
|
-
home=os.path.expanduser('~')
|
|
67
|
-
def stamp(s):
|
|
68
|
-
return datetime.datetime.fromisoformat(s.replace('Z','+00:00')).timestamp()
|
|
69
|
-
${startReader}
|
|
70
|
-
${sourceReader}
|
|
71
|
-
low,high=stamp(start),stamp(end)
|
|
72
|
-
pattern='.codex/sessions/*/*/*/rollout-*.jsonl' if harness=='codex' else '.claude/projects/*/*.jsonl'
|
|
73
|
-
result=[]
|
|
74
|
-
for count,relative in enumerate(glob.iglob(pattern,root_dir=home)):
|
|
75
|
-
if count>=100000: raise RuntimeError('listing limit')
|
|
76
|
-
path=os.path.join(home,relative)
|
|
77
|
-
if os.path.realpath(path)!=path or not stat.S_ISREG(os.stat(path).st_mode): continue
|
|
78
|
-
try:
|
|
79
|
-
with source_open(relative) as f: time=session_start(f,harness)
|
|
80
|
-
if isinstance(time,str) and low<=stamp(time)<=high: result.append({'path':relative,'start':time})
|
|
81
|
-
except (ValueError,KeyError,TypeError,OSError,RuntimeError): continue
|
|
82
|
-
if len(result)>10000: raise RuntimeError('file limit')
|
|
83
|
-
print(json.dumps(result))`;
|
|
84
|
-
function invocation(window, script) {
|
|
85
|
-
const command = quote([
|
|
86
|
-
"python3",
|
|
87
|
-
"-c",
|
|
88
|
-
script
|
|
89
|
-
]);
|
|
90
|
-
return window.transport === "ssh" ? {
|
|
91
|
-
executable: "ssh",
|
|
92
|
-
args: [
|
|
93
|
-
"-oBatchMode=yes",
|
|
94
|
-
"-oConnectTimeout=10",
|
|
95
|
-
"-oStrictHostKeyChecking=yes",
|
|
96
|
-
"-n",
|
|
97
|
-
...window.args,
|
|
98
|
-
command
|
|
99
|
-
]
|
|
100
|
-
} : {
|
|
101
|
-
executable: "gcloud",
|
|
102
|
-
args: [
|
|
103
|
-
"compute",
|
|
104
|
-
"ssh",
|
|
105
|
-
...window.args,
|
|
106
|
-
"--quiet",
|
|
107
|
-
"--ssh-flag=-oBatchMode=yes",
|
|
108
|
-
"--ssh-flag=-oConnectTimeout=10",
|
|
109
|
-
"--ssh-flag=-oStrictHostKeyChecking=yes",
|
|
110
|
-
"--ssh-flag=-n",
|
|
111
|
-
"--command",
|
|
112
|
-
command
|
|
113
|
-
]
|
|
114
|
-
};
|
|
115
|
-
}
|
|
116
|
-
async function run(window, script, maxBuffer) {
|
|
117
|
-
const call = invocation(window, script);
|
|
118
|
-
const execution = execute(call.executable, call.args, {
|
|
119
|
-
timeout: 6e4,
|
|
120
|
-
killSignal: "SIGKILL",
|
|
121
|
-
maxBuffer,
|
|
122
|
-
encoding: "buffer",
|
|
123
|
-
env: {
|
|
124
|
-
...process.env,
|
|
125
|
-
SSH_ASKPASS_REQUIRE: "never",
|
|
126
|
-
CLOUDSDK_CORE_DISABLE_PROMPTS: "1"
|
|
127
|
-
}
|
|
128
|
-
});
|
|
129
|
-
let timer;
|
|
130
|
-
const deadline = new Promise((_resolve, reject) => {
|
|
131
|
-
timer = setTimeout(() => {
|
|
132
|
-
execution.child?.kill("SIGKILL");
|
|
133
|
-
execution.child?.stdin?.destroy();
|
|
134
|
-
execution.child?.stdout?.destroy();
|
|
135
|
-
execution.child?.stderr?.destroy();
|
|
136
|
-
reject(/* @__PURE__ */ new Error("REMOTE_TRANSPORT_TIMEOUT"));
|
|
137
|
-
}, 6e4);
|
|
138
|
-
});
|
|
139
|
-
try {
|
|
140
|
-
return await Promise.race([execution, deadline]);
|
|
141
|
-
} finally {
|
|
142
|
-
clearTimeout(timer);
|
|
143
|
-
}
|
|
144
|
-
}
|
|
145
|
-
function createRemoteTransport() {
|
|
146
|
-
return {
|
|
147
|
-
async list(window) {
|
|
148
|
-
const input = JSON.stringify([
|
|
149
|
-
window.harness,
|
|
150
|
-
window.start,
|
|
151
|
-
window.end
|
|
152
|
-
]);
|
|
153
|
-
const { stdout } = await run(window, listing.replace("INPUT", input), 4194304);
|
|
154
|
-
const value = JSON.parse(stdout.toString("utf8"));
|
|
155
|
-
if (!Array.isArray(value)) throw new Error("INVALID_REMOTE_LIST");
|
|
156
|
-
const files = [];
|
|
157
|
-
for (const item of value) {
|
|
158
|
-
if (!item || typeof item !== "object" || !("path" in item) || !("start" in item) || typeof item.path !== "string" || typeof item.start !== "string") throw new Error("INVALID_REMOTE_FILE");
|
|
159
|
-
files.push({
|
|
160
|
-
path: item.path,
|
|
161
|
-
start: item.start
|
|
162
|
-
});
|
|
163
|
-
}
|
|
164
|
-
return files;
|
|
165
|
-
},
|
|
166
|
-
async copy(window, file, destination) {
|
|
167
|
-
if (!validRemotePath(file.path, window.harness)) throw new Error("INVALID_REMOTE_PATH");
|
|
168
|
-
const { stdout } = await run(window, `import os,sys,stat,json,datetime
|
|
169
|
-
def stamp(value): return datetime.datetime.fromisoformat(value.replace('Z','+00:00')).timestamp()
|
|
170
|
-
${startReader}
|
|
171
|
-
${sourceReader}
|
|
172
|
-
with source_open(${JSON.stringify(file.path)}) as f:
|
|
173
|
-
s=os.fstat(f.fileno())
|
|
174
|
-
if not stat.S_ISREG(s.st_mode) or s.st_size>268435456: raise RuntimeError('source limit')
|
|
175
|
-
time=session_start(f,${JSON.stringify(window.harness)})
|
|
176
|
-
if time is None or not stamp(${JSON.stringify(window.start)})<=stamp(time)<=stamp(${JSON.stringify(window.end)}): raise RuntimeError('source outside window')
|
|
177
|
-
f.seek(0)
|
|
178
|
-
data=f.read(268435457)
|
|
179
|
-
if len(data)>268435456: raise RuntimeError('source limit')
|
|
180
|
-
sys.stdout.buffer.write(data)`, 268435456);
|
|
181
|
-
await writeFile(destination, stdout, {
|
|
182
|
-
flag: "wx",
|
|
183
|
-
mode: 384
|
|
184
|
-
});
|
|
185
|
-
}
|
|
186
|
-
};
|
|
187
|
-
}
|
|
188
|
-
//#endregion
|
|
189
|
-
//#region apps/cli/src/remote/collect.ts
|
|
190
|
-
/** Validate the copied bounded native start records rather than trusting a stale remote listing. */
|
|
191
|
-
async function copiedStart(path, harness) {
|
|
192
|
-
const handle = await open(path, "r");
|
|
193
|
-
try {
|
|
194
|
-
if ((await handle.stat()).size > 268435456) throw new Error("REMOTE_FILE_TOO_LARGE");
|
|
195
|
-
const limit = 262144;
|
|
196
|
-
const buffer = Buffer.alloc(262145);
|
|
197
|
-
const { bytesRead } = await handle.read(buffer, 0, buffer.length, 0);
|
|
198
|
-
const lines = buffer.toString("utf8", 0, Math.min(bytesRead, limit)).split("\n");
|
|
199
|
-
if (bytesRead > limit) lines.pop();
|
|
200
|
-
const types = [
|
|
201
|
-
"user",
|
|
202
|
-
"assistant",
|
|
203
|
-
"system",
|
|
204
|
-
"progress",
|
|
205
|
-
"summary",
|
|
206
|
-
"result",
|
|
207
|
-
"queue-operation"
|
|
208
|
-
];
|
|
209
|
-
for (const line of lines) {
|
|
210
|
-
if (!line.trim()) continue;
|
|
211
|
-
const value = JSON.parse(line);
|
|
212
|
-
if (!value || typeof value !== "object" || !("type" in value)) continue;
|
|
213
|
-
if (harness === "codex" && value.type !== "session_meta") return null;
|
|
214
|
-
if (harness === "claude" && (typeof value.type !== "string" || !types.includes(value.type))) continue;
|
|
215
|
-
if ("timestamp" in value && typeof value.timestamp === "string" && Number.isFinite(Date.parse(value.timestamp))) return value.timestamp;
|
|
216
|
-
if (harness === "codex") return null;
|
|
217
|
-
}
|
|
218
|
-
return null;
|
|
219
|
-
} finally {
|
|
220
|
-
await handle.close();
|
|
221
|
-
}
|
|
222
|
-
}
|
|
223
|
-
/** Only consented hosts are contacted. Every copy is exclusive and normal discovery reads it next. */
|
|
224
|
-
async function collectRemoteWorkers(dispatches, confirmations, home, transport = createRemoteTransport(), reportFailure = () => console.error("REMOTE_WORKER_COLLECTION_FAILED"), roots = {}) {
|
|
225
|
-
const copied = [];
|
|
226
|
-
const failures = /* @__PURE__ */ new Set();
|
|
227
|
-
const seen = /* @__PURE__ */ new Set();
|
|
228
|
-
for (const window of dispatches) {
|
|
229
|
-
if (confirmations[window.host] !== true || failures.has(window.host)) continue;
|
|
230
|
-
let temporary;
|
|
231
|
-
try {
|
|
232
|
-
temporary = await mkdtemp(join(tmpdir(), "tacitt-remote-"));
|
|
233
|
-
const files = await transport.list(window);
|
|
234
|
-
if (files.length > 1e4) throw new Error("REMOTE_LIST_TOO_LARGE");
|
|
235
|
-
for (const file of files) {
|
|
236
|
-
if (!validRemotePath(file.path, window.harness) || !insideWindow(file.start, window)) continue;
|
|
237
|
-
const key = JSON.stringify([window.host, file.path]);
|
|
238
|
-
if (seen.has(key)) continue;
|
|
239
|
-
const staging = join(temporary, "worker.jsonl");
|
|
240
|
-
await transport.copy(window, file, staging);
|
|
241
|
-
const start = await copiedStart(staging, window.harness);
|
|
242
|
-
if (!start || !insideWindow(start, window)) {
|
|
243
|
-
await rm(staging, { force: true });
|
|
244
|
-
continue;
|
|
245
|
-
}
|
|
246
|
-
const hash = createHash("sha256").update(key).digest("hex");
|
|
247
|
-
const date = new Date(start).toISOString().slice(0, 10).split("-");
|
|
248
|
-
const path = window.harness === "codex" ? join(roots.codexHome ?? join(home, ".codex"), "sessions", ...date, `rollout-remote-${hash}.jsonl`) : join(roots.claudeHome ?? join(home, ".claude"), "projects", `remote-${hash.slice(0, 16)}`, `${hash}.jsonl`);
|
|
249
|
-
await mkdir(dirname(path), {
|
|
250
|
-
recursive: true,
|
|
251
|
-
mode: 448
|
|
252
|
-
});
|
|
253
|
-
try {
|
|
254
|
-
await copyFile(staging, path, constants.COPYFILE_EXCL);
|
|
255
|
-
copied.push({
|
|
256
|
-
path,
|
|
257
|
-
sourceSessionId: window.sourceSessionId
|
|
258
|
-
});
|
|
259
|
-
} catch (error) {
|
|
260
|
-
if (!(error instanceof Error) || !("code" in error) || error.code !== "EEXIST") throw error;
|
|
261
|
-
const existing = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
|
|
262
|
-
try {
|
|
263
|
-
if ((await existing.stat()).size <= 268435456) {
|
|
264
|
-
const incoming = await open(staging, "r");
|
|
265
|
-
try {
|
|
266
|
-
const [left, right] = await Promise.all([existing.readFile(), incoming.readFile()]);
|
|
267
|
-
if (left.equals(right)) copied.push({
|
|
268
|
-
path,
|
|
269
|
-
sourceSessionId: window.sourceSessionId
|
|
270
|
-
});
|
|
271
|
-
} finally {
|
|
272
|
-
await incoming.close();
|
|
273
|
-
}
|
|
274
|
-
}
|
|
275
|
-
} finally {
|
|
276
|
-
await existing.close();
|
|
277
|
-
}
|
|
278
|
-
}
|
|
279
|
-
seen.add(key);
|
|
280
|
-
await rm(staging, { force: true });
|
|
281
|
-
}
|
|
282
|
-
} catch (error) {
|
|
283
|
-
failures.add(window.host);
|
|
284
|
-
reportFailure(window.host, error);
|
|
285
|
-
} finally {
|
|
286
|
-
if (temporary) try {
|
|
287
|
-
await rm(temporary, {
|
|
288
|
-
recursive: true,
|
|
289
|
-
force: true
|
|
290
|
-
});
|
|
291
|
-
} catch (error) {
|
|
292
|
-
failures.add(window.host);
|
|
293
|
-
reportFailure(window.host, error);
|
|
294
|
-
}
|
|
295
|
-
}
|
|
296
|
-
}
|
|
297
|
-
return {
|
|
298
|
-
copied,
|
|
299
|
-
failures: [...failures]
|
|
300
|
-
};
|
|
301
|
-
}
|
|
302
|
-
//#endregion
|
|
303
|
-
//#region apps/cli/src/journey/directory.ts
|
|
304
|
-
async function journeyDirectory(stateDir, journeyId) {
|
|
305
|
-
let path = stateDir;
|
|
306
|
-
for (const name of ["journeys", journeyId]) {
|
|
307
|
-
path = join(path, name);
|
|
308
|
-
try {
|
|
309
|
-
await makePrivateDirectory(path);
|
|
310
|
-
} catch (cause) {
|
|
311
|
-
if (!(cause instanceof Error && "code" in cause && cause.code === "EEXIST")) throw cause;
|
|
312
|
-
}
|
|
313
|
-
const stat = await lstat(path, { bigint: true });
|
|
314
|
-
if (!stat.isDirectory() || stat.isSymbolicLink() || await realpath(path) !== path || (process.platform === "win32" ? !await windowsOwnerOnly(path, "directory") : stat.uid !== BigInt(process.getuid?.() ?? -1) || (stat.mode & 63n) !== 0n)) throw new ApplicationError("UNSAFE_STATE_PATH");
|
|
315
|
-
}
|
|
316
|
-
return path;
|
|
317
|
-
}
|
|
318
|
-
//#endregion
|
|
319
|
-
//#region apps/cli/src/remote/state.ts
|
|
320
|
-
const attachment = z.object({
|
|
321
|
-
path: z.string(),
|
|
322
|
-
sourceSessionId: z.string()
|
|
323
|
-
}).strict();
|
|
324
|
-
async function readRemoteState(stateDir, journeyId) {
|
|
325
|
-
const path = join(await journeyDirectory(stateDir, journeyId), "remote-workers.json");
|
|
326
|
-
try {
|
|
327
|
-
const value = JSON.parse(await readFile(path, "utf8"));
|
|
328
|
-
return z.object({
|
|
329
|
-
windows: z.array(z.object({
|
|
330
|
-
host: z.string(),
|
|
331
|
-
harness: z.enum(["codex", "claude"]),
|
|
332
|
-
transport: z.enum(["ssh", "gcloud"]),
|
|
333
|
-
args: z.array(z.string()),
|
|
334
|
-
sourceSessionId: z.string(),
|
|
335
|
-
start: z.string(),
|
|
336
|
-
end: z.string()
|
|
337
|
-
}).strict()),
|
|
338
|
-
decisions: z.record(z.string(), z.boolean()),
|
|
339
|
-
parents: z.record(z.string(), z.string()),
|
|
340
|
-
copied: z.array(attachment),
|
|
341
|
-
failures: z.array(z.string())
|
|
342
|
-
}).strict().parse(value);
|
|
343
|
-
} catch (error) {
|
|
344
|
-
if (error instanceof Error && "code" in error && error.code === "ENOENT") return null;
|
|
345
|
-
throw error;
|
|
346
|
-
}
|
|
347
|
-
}
|
|
348
|
-
async function saveRemoteState(stateDir, journeyId, state) {
|
|
349
|
-
const path = join(await journeyDirectory(stateDir, journeyId), "remote-workers.json");
|
|
350
|
-
const temporary = `${path}.${randomUUID()}`;
|
|
351
|
-
await writeFile(temporary, JSON.stringify(state), {
|
|
352
|
-
mode: 384,
|
|
353
|
-
flag: "wx"
|
|
354
|
-
});
|
|
355
|
-
await rename(temporary, path);
|
|
356
|
-
}
|
|
357
|
-
async function attachRemoteWorkers(inventory, stateDir, journeyId) {
|
|
358
|
-
const state = await readRemoteState(stateDir, journeyId);
|
|
359
|
-
const allowed = new Set(state?.copied.map((copy) => copy.path) ?? []);
|
|
360
|
-
const sessions = inventory.sessions.filter((session) => session.parts.every((part) => !isRemoteImport(part.path) || allowed.has(part.path))).flatMap((session) => {
|
|
361
|
-
const imports = (state?.copied ?? []).filter((copy) => session.parts.some((part) => part.path === copy.path));
|
|
362
|
-
const parent = imports[0]?.sourceSessionId;
|
|
363
|
-
const key = parent ? state?.parents[parent] : void 0;
|
|
364
|
-
const parents = key ? inventory.sessions.filter((candidate) => candidate.sourceKey === key && candidate.parts.every((part) => !isRemoteImport(part.path))) : [];
|
|
365
|
-
const resolved = parents.length === 1 ? parents[0] : void 0;
|
|
366
|
-
if (!resolved || imports.some((copy) => copy.sourceSessionId !== parent)) return session.parts.some((part) => isRemoteImport(part.path)) ? [] : [session];
|
|
367
|
-
return [{
|
|
368
|
-
...session,
|
|
369
|
-
remoteParentSessionId: resolved.id
|
|
370
|
-
}];
|
|
371
|
-
});
|
|
372
|
-
return {
|
|
373
|
-
...inventory,
|
|
374
|
-
sessions
|
|
375
|
-
};
|
|
376
|
-
}
|
|
377
|
-
/** Reserved importer names never enter a later Send without that Send's consent receipt. */
|
|
378
|
-
function isRemoteImport(path) {
|
|
379
|
-
return /(?:^|[/\\])rollout-remote-[a-f0-9]{64}\.jsonl$/u.test(path) || /(?:^|[/\\])projects[/\\]remote-[a-f0-9]{16}[/\\][a-f0-9]{64}\.jsonl$/u.test(path);
|
|
380
|
-
}
|
|
381
|
-
//#endregion
|
|
382
|
-
//#region apps/cli/src/journey/plan.ts
|
|
383
|
-
async function prepareSourcePlan(storage, harnesses, journeyId, sensitiveTerms, signal, period = {}, stateDir) {
|
|
384
|
-
const saved = (await storage.resources({
|
|
385
|
-
journeyId,
|
|
386
|
-
kind: "plan"
|
|
387
|
-
}))[0];
|
|
388
|
-
if (saved?.kind === "plan") return saved;
|
|
389
|
-
const discovered = await harnesses.discover(signal);
|
|
390
|
-
if (!discovered.complete) throw new ApplicationError("SCOPE_NOT_READY");
|
|
391
|
-
const linked = stateDir ? await attachRemoteWorkers(discovered, stateDir, journeyId) : discovered;
|
|
392
|
-
const inventory = filterInventoryByPeriod(linked, period);
|
|
393
|
-
if (inventory.sessions.length === 0) throw new ApplicationError("INVALID_SELECTION");
|
|
394
|
-
const families = reviewFamilies(inventory);
|
|
395
|
-
const defaults = defaultScopedSettings();
|
|
396
|
-
const settings = {
|
|
397
|
-
...defaults,
|
|
398
|
-
overall: {
|
|
399
|
-
...defaults.overall,
|
|
400
|
-
cleanup: {
|
|
401
|
-
ref: {
|
|
402
|
-
id: "native-evidence",
|
|
403
|
-
version: "2"
|
|
404
|
-
},
|
|
405
|
-
options: { assessedSessionsOnly: true }
|
|
406
|
-
}
|
|
407
|
-
},
|
|
408
|
-
workspaces: Object.fromEntries(inventory.workspaces.map((work) => [work.id, {
|
|
409
|
-
parameters: {
|
|
410
|
-
"screened-identifiers": { enabled: false },
|
|
411
|
-
"ip-addresses": { enabled: false },
|
|
412
|
-
hostnames: { enabled: false },
|
|
413
|
-
"cloud-resources": { enabled: false },
|
|
414
|
-
"mac-addresses": { enabled: false },
|
|
415
|
-
"public-key-containers": { enabled: false },
|
|
416
|
-
"geographic-coordinates": { enabled: false },
|
|
417
|
-
urls: { enabled: false },
|
|
418
|
-
"filesystem-paths": { mode: "identity-roots" }
|
|
419
|
-
},
|
|
420
|
-
optionalSteps: { "phone-numbers": true }
|
|
421
|
-
}]))
|
|
422
|
-
};
|
|
423
|
-
const configuration = freezeConfiguration(settings, inventory.sessions.flatMap((session) => {
|
|
424
|
-
const owner = families.owners.get(session.id);
|
|
425
|
-
return owner ? [{
|
|
426
|
-
sessionId: session.id,
|
|
427
|
-
workspaceId: session.workspaceId,
|
|
428
|
-
settingsOwner: {
|
|
429
|
-
sessionId: owner.id,
|
|
430
|
-
workspaceId: owner.workspaceId
|
|
431
|
-
}
|
|
432
|
-
}] : [];
|
|
433
|
-
}));
|
|
434
|
-
const resource = {
|
|
435
|
-
kind: "plan",
|
|
436
|
-
journeyId,
|
|
437
|
-
approval: {
|
|
438
|
-
runId: journeyId,
|
|
439
|
-
revision: 0,
|
|
440
|
-
approvedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
441
|
-
inventory,
|
|
442
|
-
sourceChangePolicy: "fail",
|
|
443
|
-
treatment: {
|
|
444
|
-
kind: "configured",
|
|
445
|
-
configuration,
|
|
446
|
-
dictionaryRevision: sensitiveTerms.revision
|
|
447
|
-
}
|
|
448
|
-
},
|
|
449
|
-
replacementKey: randomBytes(32).toString("hex"),
|
|
450
|
-
sensitiveTerms: {
|
|
451
|
-
revision: sensitiveTerms.revision,
|
|
452
|
-
terms: [...sensitiveTerms.terms]
|
|
453
|
-
}
|
|
454
|
-
};
|
|
455
|
-
await storage.saveResource(resource);
|
|
456
|
-
return resource;
|
|
457
|
-
}
|
|
458
|
-
function workApproval(approval, rootIds) {
|
|
459
|
-
if (approval.treatment?.kind !== "configured") throw new ApplicationError("PROCESSING_PLAN_UNAVAILABLE");
|
|
460
|
-
const families = reviewFamilies(approval.inventory);
|
|
461
|
-
const ids = new Set(rootIds.flatMap((id) => {
|
|
462
|
-
const members = families.members.get(id);
|
|
463
|
-
if (!members) throw new ApplicationError("UNKNOWN_ITEM");
|
|
464
|
-
return members.map((session) => session.id);
|
|
465
|
-
}));
|
|
466
|
-
const sessions = approval.inventory.sessions.filter((session) => ids.has(session.id));
|
|
467
|
-
const workspaces = new Set(sessions.map((session) => session.workspaceId));
|
|
468
|
-
return {
|
|
469
|
-
...approval,
|
|
470
|
-
inventory: {
|
|
471
|
-
...approval.inventory,
|
|
472
|
-
sessions,
|
|
473
|
-
workspaces: approval.inventory.workspaces.filter((workspace) => workspaces.has(workspace.id))
|
|
474
|
-
},
|
|
475
|
-
treatment: {
|
|
476
|
-
...approval.treatment,
|
|
477
|
-
configuration: selectFrozenConfiguration(approval.treatment.configuration, ids)
|
|
478
|
-
}
|
|
479
|
-
};
|
|
480
|
-
}
|
|
481
|
-
//#endregion
|
|
482
|
-
//#region apps/cli/src/journey/static/production.ts
|
|
483
|
-
/** Store reads of `staticBaseProduction`; undefined when no reviewed root remains. */
|
|
484
|
-
async function staticBaseInputs(storage, state, work) {
|
|
485
|
-
const excluded = work.excludedReview?.snapshotId === state.snapshotId && work.excludedReview.inputRevision === work.inputRevision ? new Set(work.excludedReview.rootSessionIds) : /* @__PURE__ */ new Set();
|
|
486
|
-
const roots = work.rootSessionIds.filter((id) => !excluded.has(id));
|
|
487
|
-
if (!roots.length) return void 0;
|
|
488
|
-
const plan = (await storage.resources({
|
|
489
|
-
journeyId: state.journeyId,
|
|
490
|
-
kind: "plan"
|
|
491
|
-
}))[0];
|
|
492
|
-
if (plan?.kind !== "plan") throw new ApplicationError("CORRUPTED_STATE");
|
|
493
|
-
const approval = workApproval(plan.approval, roots);
|
|
494
|
-
if (approval.treatment?.kind !== "configured") throw new ApplicationError("CORRUPTED_STATE");
|
|
495
|
-
const parts = new Set(approval.inventory.sessions.flatMap((session) => session.parts.map((part) => part.id)));
|
|
496
|
-
const captures = (await storage.resources({
|
|
497
|
-
journeyId: state.journeyId,
|
|
498
|
-
kind: "capture"
|
|
499
|
-
})).flatMap((item) => item.kind === "capture" && parts.has(item.capture.partId) ? [item.capture] : []);
|
|
500
|
-
if (captures.length !== parts.size) throw new ApplicationError("SCOPE_NOT_READY");
|
|
501
|
-
return {
|
|
502
|
-
journeyId: state.journeyId,
|
|
503
|
-
work,
|
|
504
|
-
roots,
|
|
505
|
-
approval,
|
|
506
|
-
captures,
|
|
507
|
-
replacementKey: createHash("sha256").update(JSON.stringify([
|
|
508
|
-
"static-base-v1",
|
|
509
|
-
plan.replacementKey,
|
|
510
|
-
work.id,
|
|
511
|
-
work.inputRevision
|
|
512
|
-
])).digest("hex"),
|
|
513
|
-
staticExceptions: {
|
|
514
|
-
workId: work.id,
|
|
515
|
-
values: state.exceptions.filter((item) => item.workId === work.id).map(({ kind, value }) => ({
|
|
516
|
-
kind,
|
|
517
|
-
value
|
|
518
|
-
}))
|
|
519
|
-
}
|
|
520
|
-
};
|
|
521
|
-
}
|
|
522
|
-
/** The production of `staticBaseProduction` from its stored inputs; no store access. */
|
|
523
|
-
async function staticBaseFromInputs(inputs, processing, hooks) {
|
|
524
|
-
const { work, roots, approval } = inputs;
|
|
525
|
-
const { sensitiveTerms: _live, customTerms: _terms, staticExceptions: _unscoped, ...local } = processing;
|
|
526
|
-
const production = await prepareCollectionProduction(approval, inputs.journeyId, {
|
|
527
|
-
configured: {
|
|
528
|
-
...local,
|
|
529
|
-
gitNames: async () => ({
|
|
530
|
-
names: [],
|
|
531
|
-
snapshots: []
|
|
532
|
-
}),
|
|
533
|
-
replacementKey: inputs.replacementKey,
|
|
534
|
-
staticExceptions: inputs.staticExceptions
|
|
535
|
-
},
|
|
536
|
-
captures: inputs.captures,
|
|
537
|
-
everyAdmissionRefusal: true
|
|
538
|
-
}, hooks);
|
|
539
|
-
if (production.kind !== "configured" || production.workspaceFailures.length) throw new ApplicationError("SCOPE_NOT_READY");
|
|
540
|
-
const families = reviewFamilies(approval.inventory);
|
|
541
|
-
const admitted = new Set(production.selection.sessions.map((item) => item.session.id));
|
|
542
|
-
return {
|
|
543
|
-
production,
|
|
544
|
-
families,
|
|
545
|
-
work: {
|
|
546
|
-
...work,
|
|
547
|
-
rootSessionIds: roots.filter((id) => families.members.get(id)?.some((member) => admitted.has(member.id))),
|
|
548
|
-
access: work.access === "private" ? "private" : "public"
|
|
549
|
-
}
|
|
550
|
-
};
|
|
551
|
-
}
|
|
552
|
-
/**
|
|
553
|
-
* The single production both Track B and packaging use for one Work generation (ADR 0172):
|
|
554
|
-
* frozen captures, built-in static detection only (no custom terms, no Git seeds), the Work's
|
|
555
|
-
* exceptions and a replacement key bound to this generation. Identical inputs give identical
|
|
556
|
-
* session render jobs, so packaging can require the stored Track B result by job digest.
|
|
557
|
-
*/
|
|
558
|
-
async function staticBaseProduction(storage, state, work, processing, hooks) {
|
|
559
|
-
const inputs = await staticBaseInputs(storage, state, work);
|
|
560
|
-
return inputs && staticBaseFromInputs(inputs, processing, hooks);
|
|
561
|
-
}
|
|
562
|
-
//#endregion
|
|
563
|
-
//#region apps/cli/src/journey/static/store.ts
|
|
564
|
-
/** Identity, length and complete digest of one owner-only file; a symlink is never followed. */
|
|
565
|
-
async function describe(path, signal) {
|
|
566
|
-
const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
|
|
567
|
-
try {
|
|
568
|
-
const before = await handle.stat({ bigint: true });
|
|
569
|
-
if (!before.isFile()) throw new ApplicationError("CORRUPTED_STATE");
|
|
570
|
-
const digest = createHash("sha256");
|
|
571
|
-
let byteLength = 0;
|
|
572
|
-
const buffer = Buffer.allocUnsafe(1048576);
|
|
573
|
-
for (;;) {
|
|
574
|
-
signal.throwIfAborted();
|
|
575
|
-
const { bytesRead } = await handle.read(buffer, 0, buffer.length, byteLength);
|
|
576
|
-
if (!bytesRead) break;
|
|
577
|
-
digest.update(buffer.subarray(0, bytesRead));
|
|
578
|
-
byteLength += bytesRead;
|
|
579
|
-
}
|
|
580
|
-
const after = await handle.stat({ bigint: true });
|
|
581
|
-
if (after.dev !== before.dev || after.ino !== before.ino || after.size !== before.size || after.mtimeNs !== before.mtimeNs || BigInt(byteLength) !== after.size) throw new ApplicationError("CORRUPTED_STATE");
|
|
582
|
-
return {
|
|
583
|
-
path,
|
|
584
|
-
identity: {
|
|
585
|
-
dev: String(after.dev),
|
|
586
|
-
ino: String(after.ino)
|
|
587
|
-
},
|
|
588
|
-
sha256: digest.digest("hex"),
|
|
589
|
-
byteLength
|
|
590
|
-
};
|
|
591
|
-
} finally {
|
|
592
|
-
await handle.close();
|
|
593
|
-
}
|
|
594
|
-
}
|
|
595
|
-
/** A stored file must still be the exact recorded file and bytes. */
|
|
596
|
-
async function authenticate(file, signal) {
|
|
597
|
-
const actual = await describe(file.path, signal);
|
|
598
|
-
if (JSON.stringify(actual) !== JSON.stringify(file)) throw new ApplicationError("CORRUPTED_STATE");
|
|
599
|
-
}
|
|
600
|
-
/**
|
|
601
|
-
* Persist one Track B session as owner-only private state beside the journey's other scan
|
|
602
|
-
* files: its three spools and a digest-bound result description. Nothing here is uploaded.
|
|
603
|
-
*/
|
|
604
|
-
async function saveStaticSession(storage, binding, item, result, signal) {
|
|
605
|
-
const owned = [
|
|
606
|
-
result.spoolPath,
|
|
607
|
-
result.companionSpoolPath,
|
|
608
|
-
result.maskSpoolPath
|
|
609
|
-
];
|
|
610
|
-
const resultPath = join(binding.directory, `${randomUUID()}.static.json`);
|
|
611
|
-
let saved = false;
|
|
612
|
-
try {
|
|
613
|
-
if (result.kind !== "render" || result.maskSpoolPath === void 0 || result.maskEntries === void 0 || result.metadataMasks === void 0 || result.reviewSpoolPath !== void 0) throw new ApplicationError("CORRUPTED_STATE");
|
|
614
|
-
const stored = {
|
|
615
|
-
kind: "render",
|
|
616
|
-
processingMs: result.processingMs,
|
|
617
|
-
maskEntries: result.maskEntries,
|
|
618
|
-
metadataMasks: result.metadataMasks,
|
|
619
|
-
metadata: result.metadata,
|
|
620
|
-
parts: result.parts,
|
|
621
|
-
entries: result.entries,
|
|
622
|
-
agentSetup: result.agentSetup,
|
|
623
|
-
...result.agentSources === void 0 ? {} : { agentSources: result.agentSources }
|
|
624
|
-
};
|
|
625
|
-
const file = await open(resultPath, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY | constants.O_NOFOLLOW, 384);
|
|
626
|
-
owned.push(resultPath);
|
|
627
|
-
try {
|
|
628
|
-
await file.writeFile(JSON.stringify(stored));
|
|
629
|
-
await file.sync();
|
|
630
|
-
} finally {
|
|
631
|
-
await file.close();
|
|
632
|
-
}
|
|
633
|
-
const resource = staticSessionResourceSchema.parse({
|
|
634
|
-
kind: "static-session",
|
|
635
|
-
...result.agentSources === void 0 ? {} : { agentSources: result.agentSources },
|
|
636
|
-
journeyId: binding.journeyId,
|
|
637
|
-
workId: binding.work.id,
|
|
638
|
-
inputRevision: binding.work.inputRevision,
|
|
639
|
-
snapshotId: binding.snapshotId,
|
|
640
|
-
sessionId: item.sessionId,
|
|
641
|
-
jobDigest: item.digest,
|
|
642
|
-
result: await describe(resultPath, signal),
|
|
643
|
-
primary: await describe(result.spoolPath, signal),
|
|
644
|
-
companion: await describe(result.companionSpoolPath, signal),
|
|
645
|
-
masks: await describe(result.maskSpoolPath, signal),
|
|
646
|
-
...result.images === void 0 ? {} : { imagesOmitted: result.images }
|
|
647
|
-
});
|
|
648
|
-
await storage.saveResource(resource);
|
|
649
|
-
saved = true;
|
|
650
|
-
return resource;
|
|
651
|
-
} finally {
|
|
652
|
-
if (!saved) {
|
|
653
|
-
for (const path of owned) if (path !== void 0) await unlink(path).catch(() => {});
|
|
654
|
-
}
|
|
655
|
-
}
|
|
656
|
-
}
|
|
657
|
-
/** Current-generation Track B receipts of one Work, keyed by application session ID. */
|
|
658
|
-
async function staticSessions(storage, journeyId, work) {
|
|
659
|
-
const resources = (await storage.resources({
|
|
660
|
-
journeyId,
|
|
661
|
-
kind: "static-session",
|
|
662
|
-
idPrefix: `${work.id}:${work.inputRevision}:`
|
|
663
|
-
})).filter((item) => item.kind === "static-session" && item.workId === work.id && item.inputRevision === work.inputRevision);
|
|
664
|
-
return new Map(resources.map((item) => [item.sessionId, item]));
|
|
665
|
-
}
|
|
666
|
-
/**
|
|
667
|
-
* Shape guard like the worker transport's: the bytes are bound by the recorded digest, and the
|
|
668
|
-
* archive writer checks parts, entries and masks again when it consumes them.
|
|
669
|
-
*/
|
|
670
|
-
function isStoredResult(value) {
|
|
671
|
-
return typeof value === "object" && value !== null && Reflect.get(value, "kind") === "render" && Array.isArray(Reflect.get(value, "maskEntries")) && Array.isArray(Reflect.get(value, "metadataMasks")) && Array.isArray(Reflect.get(value, "parts")) && Array.isArray(Reflect.get(value, "entries")) && Array.isArray(Reflect.get(value, "agentSetup")) && typeof Reflect.get(value, "metadata") === "string" && Number.isSafeInteger(Reflect.get(value, "processingMs"));
|
|
672
|
-
}
|
|
673
|
-
/** Authenticated result description of one stored Track B session. */
|
|
674
|
-
async function readStaticResult(resource, signal) {
|
|
675
|
-
await authenticate(resource.result, signal);
|
|
676
|
-
const handle = await open(resource.result.path, constants.O_RDONLY | constants.O_NOFOLLOW);
|
|
677
|
-
try {
|
|
678
|
-
const text = await handle.readFile("utf8");
|
|
679
|
-
if (createHash("sha256").update(text).digest("hex") !== resource.result.sha256) throw new ApplicationError("CORRUPTED_STATE");
|
|
680
|
-
const value = JSON.parse(text);
|
|
681
|
-
if (!isStoredResult(value)) throw new ApplicationError("CORRUPTED_STATE");
|
|
682
|
-
return value;
|
|
683
|
-
} finally {
|
|
684
|
-
await handle.close();
|
|
685
|
-
}
|
|
686
|
-
}
|
|
687
|
-
/**
|
|
688
|
-
* Packaging from the Track B base: the stored result of exactly this job, with fresh hard links
|
|
689
|
-
* to its authenticated spools that the archive writer may consume. Static detection never runs.
|
|
690
|
-
*/
|
|
691
|
-
async function linkedStaticResult(resource, job, signal) {
|
|
692
|
-
if (!resource || resource.jobDigest !== job.digest) throw new ApplicationError("SCOPE_NOT_READY");
|
|
693
|
-
const result = await readStaticResult(resource, signal);
|
|
694
|
-
const links = [];
|
|
695
|
-
try {
|
|
696
|
-
const linked = async (file) => {
|
|
697
|
-
const path = join(job.spoolDirectory, `${randomUUID()}.session`);
|
|
698
|
-
await link(file.path, path);
|
|
699
|
-
links.push(path);
|
|
700
|
-
const stat = await lstat(path, { bigint: true });
|
|
701
|
-
if (String(stat.dev) !== file.identity.dev || String(stat.ino) !== file.identity.ino) throw new ApplicationError("CORRUPTED_STATE");
|
|
702
|
-
await authenticate({
|
|
703
|
-
...file,
|
|
704
|
-
path
|
|
705
|
-
}, signal);
|
|
706
|
-
return path;
|
|
707
|
-
};
|
|
708
|
-
return {
|
|
709
|
-
...result,
|
|
710
|
-
spoolPath: await linked(resource.primary),
|
|
711
|
-
companionSpoolPath: await linked(resource.companion),
|
|
712
|
-
maskSpoolPath: await linked(resource.masks)
|
|
713
|
-
};
|
|
714
|
-
} catch (error) {
|
|
715
|
-
for (const path of links) await unlink(path).catch(() => {});
|
|
716
|
-
throw error;
|
|
717
|
-
}
|
|
718
|
-
}
|
|
719
|
-
//#endregion
|
|
720
|
-
//#region apps/cli/src/journey/render/base.ts
|
|
721
|
-
/**
|
|
722
|
-
* The CPU-bound part of base packaging (ADR 0172): the static production from frozen captures,
|
|
723
|
-
* the base pair written from the stored Track B results and its full readback. The owning
|
|
724
|
-
* thread keeps every store read and write; this runs there or in a packaging worker.
|
|
725
|
-
*/
|
|
726
|
-
async function writeJourneyBase(work, signal, onPath) {
|
|
727
|
-
const { inputs } = work;
|
|
728
|
-
const hooks = {
|
|
729
|
-
signal,
|
|
730
|
-
onTemporary: async () => {},
|
|
731
|
-
onPhase: async () => {}
|
|
732
|
-
};
|
|
733
|
-
const prepared = await staticBaseFromInputs(inputs, work.processing, hooks);
|
|
734
|
-
const stored = new Map(work.stored.map((item) => [item.sessionId, item]));
|
|
735
|
-
const written = await writeLogicalJourneyBase({
|
|
736
|
-
production: prepared.production,
|
|
737
|
-
directory: work.directory,
|
|
738
|
-
captureReview: false,
|
|
739
|
-
work: prepared.work,
|
|
740
|
-
prerendered: async (job) => {
|
|
741
|
-
if (job.spoolDirectory === void 0) throw new ApplicationError("CORRUPTED_STATE");
|
|
742
|
-
return linkedStaticResult(stored.get(job.session.id), {
|
|
743
|
-
digest: renderJobDigest(job),
|
|
744
|
-
spoolDirectory: job.spoolDirectory
|
|
745
|
-
}, signal);
|
|
746
|
-
}
|
|
747
|
-
}, hooks, onPath);
|
|
748
|
-
return {
|
|
749
|
-
kind: "base",
|
|
750
|
-
journeyId: inputs.journeyId,
|
|
751
|
-
workId: inputs.work.id,
|
|
752
|
-
inputRevision: inputs.work.inputRevision,
|
|
753
|
-
bundleId: written.bundleId,
|
|
754
|
-
primary: written.primary,
|
|
755
|
-
companion: written.companion,
|
|
756
|
-
maskLedger: written.maskLedger,
|
|
757
|
-
sourceBindings: [...written.sourceBindings],
|
|
758
|
-
sessions: written.sessionBindings.map((binding) => {
|
|
759
|
-
const root = prepared.families.owners.get(binding.applicationSessionId);
|
|
760
|
-
if (!root) throw new ApplicationError("CORRUPTED_STATE");
|
|
761
|
-
return {
|
|
762
|
-
applicationSessionId: binding.applicationSessionId,
|
|
763
|
-
archiveSessionId: binding.sessionId,
|
|
764
|
-
rootSessionId: root.id,
|
|
765
|
-
role: root.id === binding.applicationSessionId ? "main" : "worker"
|
|
766
|
-
};
|
|
767
|
-
})
|
|
768
|
-
};
|
|
769
|
-
}
|
|
770
|
-
function journeyBaseJob(work) {
|
|
771
|
-
const { processing, ...rest } = work;
|
|
772
|
-
return {
|
|
773
|
-
...rest,
|
|
774
|
-
processing: {
|
|
775
|
-
environment: processing.environment,
|
|
776
|
-
...processing.limits === void 0 ? {} : { limits: processing.limits }
|
|
777
|
-
}
|
|
778
|
-
};
|
|
779
|
-
}
|
|
780
|
-
/**
|
|
781
|
-
* Worker side: `live` holds the worker's own adapters and static processor. The base runs no
|
|
782
|
-
* static detection and reads no Git names (ADR 0172); a Git-name read fails closed.
|
|
783
|
-
*/
|
|
784
|
-
async function runJourneyBaseJob(job, live, signal, onPath) {
|
|
785
|
-
const processing = {
|
|
786
|
-
...live,
|
|
787
|
-
...job.processing,
|
|
788
|
-
gitNames: gitSeedsProvider([])
|
|
789
|
-
};
|
|
790
|
-
return {
|
|
791
|
-
kind: "journey-base",
|
|
792
|
-
base: await writeJourneyBase({
|
|
793
|
-
...job,
|
|
794
|
-
processing
|
|
795
|
-
}, signal, onPath)
|
|
796
|
-
};
|
|
797
|
-
}
|
|
798
|
-
//#endregion
|
|
799
|
-
//#region apps/cli/src/journey/hide-index.ts
|
|
800
|
-
/**
|
|
801
|
-
* Word pieces of render input, gathered as it is read so its texts are not held to the end.
|
|
802
|
-
* `from` continues a collection begun in another thread (ADR 0181).
|
|
803
|
-
*/
|
|
804
|
-
function createHideIndexTokens(from) {
|
|
805
|
-
const tokens = new Set(from?.tokens);
|
|
806
|
-
let complete = from?.complete ?? true;
|
|
807
|
-
return {
|
|
808
|
-
tokens,
|
|
809
|
-
add: (text) => {
|
|
810
|
-
if (complete) complete = addHideIndexTokens(text, (token) => tokens.add(token));
|
|
811
|
-
if (tokens.size > 2e6) complete = false;
|
|
812
|
-
},
|
|
813
|
-
complete: () => complete
|
|
814
|
-
};
|
|
815
|
-
}
|
|
816
|
-
/**
|
|
817
|
-
* The pieces of a Work collected in shards (ADR 0199): the same set and completeness as one
|
|
818
|
-
* collection, since pieces only accumulate and the bound applies to their union.
|
|
819
|
-
*/
|
|
820
|
-
function mergeHideIndexTokens(parts) {
|
|
821
|
-
const tokens = /* @__PURE__ */ new Set();
|
|
822
|
-
let complete = true;
|
|
823
|
-
for (const part of parts) {
|
|
824
|
-
complete &&= part.complete;
|
|
825
|
-
for (const token of part.tokens) {
|
|
826
|
-
if (!complete) break;
|
|
827
|
-
tokens.add(token);
|
|
828
|
-
if (tokens.size > 2e6) complete = false;
|
|
829
|
-
}
|
|
830
|
-
}
|
|
831
|
-
return createHideIndexTokens({
|
|
832
|
-
tokens: complete ? tokens : [],
|
|
833
|
-
complete
|
|
834
|
-
});
|
|
835
|
-
}
|
|
836
|
-
/** The Work's own texts a render can mask, read before its base archive. */
|
|
837
|
-
function hideIndexTexts(work) {
|
|
838
|
-
const suggestion = work.suggestion;
|
|
839
|
-
return [
|
|
840
|
-
work.title,
|
|
841
|
-
work.description,
|
|
842
|
-
work.reason,
|
|
843
|
-
...suggestion ? [
|
|
844
|
-
suggestion.title,
|
|
845
|
-
suggestion.description,
|
|
846
|
-
suggestion.reason
|
|
847
|
-
] : []
|
|
848
|
-
];
|
|
849
|
-
}
|
|
850
|
-
/**
|
|
851
|
-
* The CPU-bound part of a hide index: word pieces of the texts and every base field, or of one
|
|
852
|
-
* shard's base fields (ADR 0199).
|
|
853
|
-
*/
|
|
854
|
-
async function collectHideIndexTokens(texts, primary, processing, { add, complete }, signal, shard) {
|
|
855
|
-
for (const text of texts) add(text);
|
|
856
|
-
for await (const field of readConfiguredSearchFields(primary, processing, signal, true, void 0, shard)) {
|
|
857
|
-
if (!complete()) break;
|
|
858
|
-
add(field.text);
|
|
859
|
-
}
|
|
860
|
-
}
|
|
861
|
-
/** Keyed digests of collected word pieces and of the confirmed Hide values. */
|
|
862
|
-
async function packRenderHideIndex(input) {
|
|
863
|
-
const { work } = input;
|
|
864
|
-
const { tokens, complete } = input.tokens;
|
|
865
|
-
const key = await input.storage.hideIndexKey(input.journeyId);
|
|
866
|
-
return {
|
|
867
|
-
kind: "hide-index",
|
|
868
|
-
journeyId: input.journeyId,
|
|
869
|
-
workId: work.id,
|
|
870
|
-
revision: work.revision,
|
|
871
|
-
confirmedHideRevision: input.confirmedHideRevision,
|
|
872
|
-
keyId: hideKeyId(key),
|
|
873
|
-
digests: complete() ? packHideIndex(key, tokens) : null,
|
|
874
|
-
values: packHideIndex(key, [...input.confirmed].map(([term, { kind }]) => renderedHideValue(kind, term)))
|
|
875
|
-
};
|
|
876
|
-
}
|
|
877
|
-
//#endregion
|
|
878
|
-
//#region apps/cli/src/journey/render/job.ts
|
|
879
|
-
/**
|
|
880
|
-
* The same steps on the owning thread; also the fallback when workers cannot run. The written
|
|
881
|
-
* primary is read back as the publisher reads an existing output.
|
|
882
|
-
*/
|
|
883
|
-
async function writeJourneyOutput(work, signal, onPath) {
|
|
884
|
-
const output = await writeLogicalJourneyOutput(work.request, signal, onPath);
|
|
885
|
-
await readConfiguredOutputInfo(output.primary, signal);
|
|
886
|
-
await collectHideIndexTokens(work.texts, work.request.primary, work.request.processing, work.tokens, signal);
|
|
887
|
-
return {
|
|
888
|
-
output,
|
|
889
|
-
tokens: work.tokens
|
|
890
|
-
};
|
|
891
|
-
}
|
|
892
|
-
function journeyOutputJob(work) {
|
|
893
|
-
const { processing, ...request } = work.request;
|
|
894
|
-
return {
|
|
895
|
-
request,
|
|
896
|
-
processing: {
|
|
897
|
-
environment: processing.environment,
|
|
898
|
-
...processing.limits === void 0 ? {} : { limits: processing.limits },
|
|
899
|
-
...processing.replacementKey === void 0 ? {} : { replacementKey: processing.replacementKey },
|
|
900
|
-
...processing.staticExceptions === void 0 ? {} : { staticExceptions: processing.staticExceptions }
|
|
901
|
-
},
|
|
902
|
-
texts: work.texts,
|
|
903
|
-
tokens: [...work.tokens.tokens],
|
|
904
|
-
tokensComplete: work.tokens.complete()
|
|
905
|
-
};
|
|
906
|
-
}
|
|
907
|
-
function workerProcessing(job, live) {
|
|
908
|
-
return {
|
|
909
|
-
...live,
|
|
910
|
-
...job.processing,
|
|
911
|
-
gitNames: gitSeedsProvider([])
|
|
912
|
-
};
|
|
913
|
-
}
|
|
914
|
-
/**
|
|
915
|
-
* Worker side of one shard: its entries rewritten into private archives and the hide-index word
|
|
916
|
-
* pieces of its base entries. The first shard also takes the Work's texts, title pieces and the
|
|
917
|
-
* base manifest's fields, so the shards' pieces together are those of one job.
|
|
918
|
-
*/
|
|
919
|
-
async function runJourneyOutputShardJob(job, live, signal, onPath) {
|
|
920
|
-
const processing = workerProcessing(job, live);
|
|
921
|
-
const { maskEvidence, ...request } = job.request;
|
|
922
|
-
if (!maskEvidence) throw new Error("ARTIFACT_INVALID_APPROVAL");
|
|
923
|
-
const pieces = await writeOutputShard({
|
|
924
|
-
...request,
|
|
925
|
-
processing,
|
|
926
|
-
maskEvidence,
|
|
927
|
-
shard: job.shard
|
|
928
|
-
}, signal, onPath);
|
|
929
|
-
const first = job.shard.index === 0;
|
|
930
|
-
const tokens = createHideIndexTokens(first ? {
|
|
931
|
-
tokens: job.tokens,
|
|
932
|
-
complete: job.tokensComplete
|
|
933
|
-
} : void 0);
|
|
934
|
-
await collectHideIndexTokens(first ? job.texts : [], job.request.primary, processing, tokens, signal, job.shard);
|
|
935
|
-
return {
|
|
936
|
-
kind: "journey-output-shard",
|
|
937
|
-
pieces,
|
|
938
|
-
tokens: [...tokens.tokens],
|
|
939
|
-
tokensComplete: tokens.complete()
|
|
940
|
-
};
|
|
941
|
-
}
|
|
942
|
-
/** Worker side of one shard of a merged output's readback. */
|
|
943
|
-
async function runJourneyOutputVerifyJob(job, signal) {
|
|
944
|
-
const ledger = job.output.finalMaskLedger;
|
|
945
|
-
if (!ledger) throw new Error("ARTIFACT_INVALID_APPROVAL");
|
|
946
|
-
await verifyOutputShard({
|
|
947
|
-
...job.output,
|
|
948
|
-
finalMaskLedger: ledger
|
|
949
|
-
}, job.shard, signal);
|
|
950
|
-
return { kind: "journey-output-verify" };
|
|
951
|
-
}
|
|
952
|
-
/**
|
|
953
|
-
* Worker side: `live` holds the worker's own adapters and static processor. A job with
|
|
954
|
-
* `request.shards` merges them; its caller then reads the output back in shards (ADR 0199).
|
|
955
|
-
*/
|
|
956
|
-
async function runJourneyOutputJob(job, live, signal, onPath) {
|
|
957
|
-
const processing = workerProcessing(job, live);
|
|
958
|
-
if (job.request.shards) {
|
|
959
|
-
const output = await writeLogicalJourneyOutput({
|
|
960
|
-
...job.request,
|
|
961
|
-
processing
|
|
962
|
-
}, signal, onPath);
|
|
963
|
-
await readConfiguredOutputInfo(output.primary, signal);
|
|
964
|
-
return {
|
|
965
|
-
kind: "journey-output",
|
|
966
|
-
output,
|
|
967
|
-
tokens: [],
|
|
968
|
-
tokensComplete: true
|
|
969
|
-
};
|
|
970
|
-
}
|
|
971
|
-
const written = await writeJourneyOutput({
|
|
972
|
-
request: {
|
|
973
|
-
...job.request,
|
|
974
|
-
processing
|
|
975
|
-
},
|
|
976
|
-
texts: job.texts,
|
|
977
|
-
tokens: createHideIndexTokens({
|
|
978
|
-
tokens: job.tokens,
|
|
979
|
-
complete: job.tokensComplete
|
|
980
|
-
})
|
|
981
|
-
}, signal, onPath);
|
|
982
|
-
return {
|
|
983
|
-
kind: "journey-output",
|
|
984
|
-
output: written.output,
|
|
985
|
-
tokens: [...written.tokens.tokens],
|
|
986
|
-
tokensComplete: written.tokens.complete()
|
|
987
|
-
};
|
|
988
|
-
}
|
|
989
|
-
//#endregion
|
|
990
|
-
export { journeyDirectory as C, saveRemoteState as S, staticBaseProduction as _, writeJourneyOutput as a, isRemoteImport as b, mergeHideIndexTokens as c, runJourneyBaseJob as d, writeJourneyBase as f, staticBaseInputs as g, staticSessions as h, runJourneyOutputVerifyJob as i, packRenderHideIndex as l, saveStaticSession as m, runJourneyOutputJob as n, createHideIndexTokens as o, readStaticResult as p, runJourneyOutputShardJob as r, hideIndexTexts as s, journeyOutputJob as t, journeyBaseJob as u, prepareSourcePlan as v, collectRemoteWorkers as w, readRemoteState as x, workApproval as y };
|