@neta-art/cohub-cli 6.12.0 → 7.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -1
- package/dist/commands/apps.js +3 -1
- package/dist/commands/runtime.d.ts +4 -0
- package/dist/commands/runtime.js +117 -0
- package/dist/commands/sandboxd-binary.js +2 -2
- package/dist/commands/spaces.js +3 -0
- package/dist/index.js +3 -3
- package/dist/runtime/archive-store.d.ts +44 -0
- package/dist/runtime/archive-store.js +353 -0
- package/dist/runtime/codex-usage.d.ts +9 -0
- package/dist/runtime/codex-usage.js +23 -0
- package/dist/runtime/connection.d.ts +16 -0
- package/dist/runtime/connection.js +321 -0
- package/dist/runtime/harness.d.ts +19 -0
- package/dist/runtime/harness.js +403 -0
- package/dist/runtime/json-rpc.d.ts +37 -0
- package/dist/runtime/json-rpc.js +165 -0
- package/dist/runtime/model-catalog.d.ts +6 -0
- package/dist/runtime/model-catalog.js +19 -0
- package/dist/runtime/native-archive.d.ts +16 -0
- package/dist/runtime/native-archive.js +88 -0
- package/dist/runtime/process-group.d.ts +3 -0
- package/dist/runtime/process-group.js +68 -0
- package/dist/runtime/session-store.d.ts +47 -0
- package/dist/runtime/session-store.js +300 -0
- package/package.json +11 -4
- package/dist/commands/sandbox.d.ts +0 -3
- package/dist/commands/sandbox.js +0 -175
package/README.md
CHANGED
|
@@ -79,6 +79,23 @@ cohub -s <spaceId> spaces prompt "message" --json
|
|
|
79
79
|
COHUB_SPACE_ID=<spaceId> cohub spaces prompt "message" --json
|
|
80
80
|
```
|
|
81
81
|
|
|
82
|
+
## Local Runtime
|
|
83
|
+
|
|
84
|
+
Connect one local workspace to a Space and select Pi or Codex per turn.
|
|
85
|
+
|
|
86
|
+
```bash
|
|
87
|
+
cohub runtime up ./project --harness pi --harness codex
|
|
88
|
+
cohub runtime up ./project --space <spaceId> --harness codex
|
|
89
|
+
cohub -s <spaceId> spaces prompt "Continue" --harness codex
|
|
90
|
+
cohub runtime status --space <spaceId>
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
The Runtime uses WebSockets, native Pi RPC / Codex app-server, and existing cloud
|
|
94
|
+
streaming. Local executables and credentials are required. `sandbox up` has been
|
|
95
|
+
removed. Reconnection and result reconciliation are automatic; no recovery command is needed.
|
|
96
|
+
When a result genuinely cannot be determined, the affected Chat offers an explicit stop
|
|
97
|
+
confirmation. See [Runtime details](../../docs/local-runtime.md).
|
|
98
|
+
|
|
82
99
|
## Chats and prompts
|
|
83
100
|
|
|
84
101
|
Use `spaces prompt` for immediate sends, delayed sends, one-time schedules, recurring schedules, new Chats, and existing Chats.
|
|
@@ -357,7 +374,7 @@ legacy bare targets. A bare target checks the current Space for a file first, th
|
|
|
357
374
|
falls back to the same App references as `apps get`. Showing a preview is
|
|
358
375
|
idempotent: repeating it re-activates the same tab and refreshes any launch state
|
|
359
376
|
carried by the reference. With `--call`, the command waits for the App to announce readiness, invokes the method,
|
|
360
|
-
and waits for the App to complete the same UI command with `client.
|
|
377
|
+
and waits for the App to complete the same UI command with `client.desktop.reportResult()`.
|
|
361
378
|
|
|
362
379
|
App authors decide what is callable by registering handlers inside the App:
|
|
363
380
|
|
package/dist/commands/apps.js
CHANGED
|
@@ -508,7 +508,7 @@ export function registerApps(program) {
|
|
|
508
508
|
.option("--app-scope <scope>", "Scope granted directly to the app runtime (space.view, session.view, file.view, file.edit, taskrun.view, session.prompt.readonly, session.prompt.fullaccess, command.execute)", collectOption, [])
|
|
509
509
|
.option("--viewer-scope <scope>", "Deprecated: viewer grants are no longer gated by the app configuration", collectOption, [])
|
|
510
510
|
.option("--clear-app-scopes", "Clear app runtime scopes")
|
|
511
|
-
.option("--clear-viewer-scopes", "
|
|
511
|
+
.option("--clear-viewer-scopes", "Deprecated: clear legacy scope metadata")
|
|
512
512
|
.option("--meta <json>", "App metadata as a JSON object")
|
|
513
513
|
.option("--hide-cohub-bar", "Hide the Cohub footer bar on the public app page")
|
|
514
514
|
.option("--show-cohub-bar", "Show the Cohub footer bar on the public app page")
|
|
@@ -728,6 +728,7 @@ export function registerApps(program) {
|
|
|
728
728
|
.description("Grant an app scopes as the current user")
|
|
729
729
|
.requiredOption("--scope <scope>", "Scope to grant (repeatable)", collectOption, [])
|
|
730
730
|
.option("--space <spaceId>", "Target space; defaults to the app's own space")
|
|
731
|
+
.option("--extend", "Add scopes without replacing active grants")
|
|
731
732
|
.option("--json", "Output as JSON")
|
|
732
733
|
.action(async (appRef, opts) => {
|
|
733
734
|
const client = createClient();
|
|
@@ -736,6 +737,7 @@ export function registerApps(program) {
|
|
|
736
737
|
const result = await client.apps.authorize(detail.app.id, {
|
|
737
738
|
scopes: opts.scope,
|
|
738
739
|
...(opts.space ? { spaceId: opts.space } : {}),
|
|
740
|
+
...(opts.extend ? { scopeMode: "extend" } : {}),
|
|
739
741
|
});
|
|
740
742
|
if (jsonRequested(opts))
|
|
741
743
|
return outJson(result);
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import type { Command } from "commander";
|
|
2
|
+
export declare const resolveLocalSpaceName: (root: string, name?: string) => string;
|
|
3
|
+
export declare function parseRuntimeHarnesses(values: string[]): ("pi" | "codex")[];
|
|
4
|
+
export declare function registerRuntime(program: Command): void;
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
import { spawn } from "node:child_process";
|
|
2
|
+
import { stat } from "node:fs/promises";
|
|
3
|
+
import { basename, resolve } from "node:path";
|
|
4
|
+
import { createInterface } from "node:readline/promises";
|
|
5
|
+
import { isLocalHarness, resolveCohubEnvironment, resolveWebsocketUrl } from "@neta-art/cohub";
|
|
6
|
+
import { requireAccessToken } from "../auth.js";
|
|
7
|
+
import { createClient } from "../client.js";
|
|
8
|
+
import { error, json as outJson, jsonRequested } from "../output.js";
|
|
9
|
+
import { resolveSpace } from "../space.js";
|
|
10
|
+
import { discoverHarnesses } from "../runtime/harness.js";
|
|
11
|
+
import { serveRuntime } from "../runtime/connection.js";
|
|
12
|
+
import { RuntimeSessionStore } from "../runtime/session-store.js";
|
|
13
|
+
import { ensureSandboxdBinary } from "./sandboxd-binary.js";
|
|
14
|
+
export const resolveLocalSpaceName = (root, name) => name?.trim() || basename(root) || "local-space";
|
|
15
|
+
export function parseRuntimeHarnesses(values) {
|
|
16
|
+
const names = values.flatMap((value) => value.split(",")).map((name) => name.trim()).filter(Boolean);
|
|
17
|
+
if (names.some((name) => !isLocalHarness(name)))
|
|
18
|
+
throw new Error("Harness must be pi or codex");
|
|
19
|
+
return [...new Set(names.length ? names : ["pi"])];
|
|
20
|
+
}
|
|
21
|
+
export function registerRuntime(program) {
|
|
22
|
+
const runtime = program.command("runtime").description("Connect a local workspace");
|
|
23
|
+
runtime.command("up [dir]")
|
|
24
|
+
.description("Connect local Harnesses and files")
|
|
25
|
+
.option("-s, --space <id>", "Target Space")
|
|
26
|
+
.option("-n, --name <name>", "New Space name")
|
|
27
|
+
.option("--harness <name>", "Pi or Codex; repeatable", (value, previous) => [...previous, value], [])
|
|
28
|
+
.option("--pi <path>", "Pi executable")
|
|
29
|
+
.option("--codex <path>", "Codex executable")
|
|
30
|
+
.option("-y, --yes", "Accept local execution access")
|
|
31
|
+
.option("--json", "JSON output")
|
|
32
|
+
.action(async (dir, options) => {
|
|
33
|
+
const controller = new AbortController();
|
|
34
|
+
const stop = () => controller.abort();
|
|
35
|
+
process.once("SIGINT", stop);
|
|
36
|
+
process.once("SIGTERM", stop);
|
|
37
|
+
try {
|
|
38
|
+
const root = resolve(dir ?? process.cwd());
|
|
39
|
+
if (!(await stat(root)).isDirectory())
|
|
40
|
+
throw new Error("Workspace is not a directory");
|
|
41
|
+
const harnesses = parseRuntimeHarnesses(options.harness);
|
|
42
|
+
if (!options.yes) {
|
|
43
|
+
if (!process.stdin.isTTY)
|
|
44
|
+
throw new Error("Use --yes to authorize local execution");
|
|
45
|
+
const rl = createInterface({ input: process.stdin, output: process.stderr });
|
|
46
|
+
try {
|
|
47
|
+
const answer = await rl.question(`Connect ${root}? Space collaborators can run commands as your OS user, beyond this folder. [y/N] `);
|
|
48
|
+
if (!/^y(es)?$/i.test(answer.trim()))
|
|
49
|
+
return;
|
|
50
|
+
}
|
|
51
|
+
finally {
|
|
52
|
+
rl.close();
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
const capabilities = await discoverHarnesses(harnesses, options, root);
|
|
56
|
+
const client = createClient();
|
|
57
|
+
const requested = options.space?.trim() || program.opts().space?.trim();
|
|
58
|
+
const spaceId = requested || (await client.spaces.create({ name: resolveLocalSpaceName(root, options.name), config: { sandbox: { provider: "local" } } })).space.id;
|
|
59
|
+
const sandbox = (await client.space(spaceId).sandbox.get()).sandbox;
|
|
60
|
+
if (sandbox?.provider !== "local")
|
|
61
|
+
throw new Error("Space does not have a local Runtime");
|
|
62
|
+
const binary = await ensureSandboxdBinary();
|
|
63
|
+
const wsBase = resolveWebsocketUrl({ url: process.env.COHUB_WS_URL });
|
|
64
|
+
const url = new URL(wsBase);
|
|
65
|
+
url.pathname = "/runtime/relay";
|
|
66
|
+
const relay = new URL(wsBase);
|
|
67
|
+
relay.pathname = "/sandbox/relay";
|
|
68
|
+
let bridge = null;
|
|
69
|
+
let bridgeClosed = Promise.resolve();
|
|
70
|
+
const token = await requireAccessToken();
|
|
71
|
+
try {
|
|
72
|
+
await serveRuntime({
|
|
73
|
+
spaceId, cwd: root, url: url.toString(), capabilities, harnesses: options,
|
|
74
|
+
token: requireAccessToken, signal: controller.signal, store: new RuntimeSessionStore(spaceId, undefined, client.space(spaceId)),
|
|
75
|
+
onReady: () => {
|
|
76
|
+
if (!bridge) {
|
|
77
|
+
bridge = spawn(binary, ["--local", "--space", spaceId, "--root", root, "--relay", process.env.COHUB_RELAY_URL?.trim() || relay.toString()], { stdio: ["ignore", "inherit", "inherit"], env: { ...process.env, COHUB_RELAY_TOKEN: token } });
|
|
78
|
+
bridgeClosed = new Promise((resolveClosed) => bridge?.once("close", () => resolveClosed()));
|
|
79
|
+
bridge.on("error", (cause) => { console.error(cause); controller.abort(); });
|
|
80
|
+
bridge.once("exit", () => controller.abort());
|
|
81
|
+
}
|
|
82
|
+
const webUrl = `${resolveCohubEnvironment() === "prod" ? "https://cohub.live" : "https://dev.cohub.live"}/spaces/${spaceId}`;
|
|
83
|
+
if (jsonRequested(options))
|
|
84
|
+
outJson({ spaceId, root, harnesses, url: webUrl });
|
|
85
|
+
else
|
|
86
|
+
console.error(`Runtime connected: ${webUrl}`);
|
|
87
|
+
},
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
finally {
|
|
91
|
+
if (bridge) {
|
|
92
|
+
const child = bridge;
|
|
93
|
+
child.kill("SIGTERM");
|
|
94
|
+
const timeout = setTimeout(() => child.kill("SIGKILL"), 3000);
|
|
95
|
+
await bridgeClosed;
|
|
96
|
+
clearTimeout(timeout);
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
catch (cause) {
|
|
101
|
+
if (!controller.signal.aborted)
|
|
102
|
+
error("Runtime failed", cause instanceof Error ? cause.message : String(cause));
|
|
103
|
+
}
|
|
104
|
+
finally {
|
|
105
|
+
process.removeListener("SIGINT", stop);
|
|
106
|
+
process.removeListener("SIGTERM", stop);
|
|
107
|
+
}
|
|
108
|
+
});
|
|
109
|
+
runtime.command("status").description("Runtime status").option("-s, --space <id>", "Target Space").action(async (options) => {
|
|
110
|
+
const spaceId = options.space?.trim() || await resolveSpace(program);
|
|
111
|
+
const { archives } = new RuntimeSessionStore(spaceId);
|
|
112
|
+
const [status, pendingLocalArchives, failedLocalArchives] = await Promise.all([
|
|
113
|
+
createClient().space(spaceId).getRuntime(), archives.pendingCount(), archives.failedCaptureCount(),
|
|
114
|
+
]);
|
|
115
|
+
outJson({ ...status, pendingLocalArchives, failedLocalArchives });
|
|
116
|
+
});
|
|
117
|
+
}
|
|
@@ -12,7 +12,7 @@ import { Readable } from "node:stream";
|
|
|
12
12
|
//
|
|
13
13
|
// IMPORTANT: this must point at a tag whose CDN artifacts have already been
|
|
14
14
|
// published by .github/workflows/sandbox-binaries-build.yml. Only bump it AFTER
|
|
15
|
-
// that tag's publish-cdn job has succeeded, otherwise `
|
|
15
|
+
// that tag's publish-cdn job has succeeded, otherwise `runtime up` 404s on the
|
|
16
16
|
// default download.
|
|
17
17
|
export const SANDBOXD_VERSION = "v1.82.4";
|
|
18
18
|
const BINARY_NAME = "cohub-sandboxd";
|
|
@@ -117,7 +117,7 @@ const extractTarGz = (archivePath, cwd) => new Promise((res, rej) => {
|
|
|
117
117
|
child.on("close", (code) => code === 0 ? res() : rej(new SandboxdDownloadError(`tar extraction failed: ${stderr.trim() || `exit ${code}`}`)));
|
|
118
118
|
});
|
|
119
119
|
// Cross-process lock via atomic mkdir, mirroring the CLI self-update lock so two
|
|
120
|
-
// concurrent `
|
|
120
|
+
// concurrent `runtime up` invocations don't download the same archive twice.
|
|
121
121
|
const withLock = async (version, fn) => {
|
|
122
122
|
const lockPath = join(cacheDir(version), ".download.lock");
|
|
123
123
|
await mkdir(dirname(lockPath), { recursive: true });
|
package/dist/commands/spaces.js
CHANGED
|
@@ -282,6 +282,7 @@ async function sendPrompt(command, words, opts) {
|
|
|
282
282
|
content: promptContent,
|
|
283
283
|
model: opts.model,
|
|
284
284
|
provider: opts.provider,
|
|
285
|
+
harness: opts.harness,
|
|
285
286
|
thinkingLevel: thinkingLevel,
|
|
286
287
|
accessMode: opts.readOnly ? "read_only" : "full_access",
|
|
287
288
|
intent: opts.steer ? "steer" : undefined,
|
|
@@ -388,6 +389,7 @@ export function registerPrompt(program) {
|
|
|
388
389
|
.option("--title <title>", "Title for a newly created session or schedule")
|
|
389
390
|
.option("-m, --model <model>", "Model name")
|
|
390
391
|
.option("-p, --provider <provider>", "Provider name")
|
|
392
|
+
.option("--harness <harness>", "Harness: cohub, pi, or codex", "cohub")
|
|
391
393
|
.option("--thinking-level <level>", "Thinking level: off|minimal|low|medium|high|xhigh|max")
|
|
392
394
|
.option("--read-only", "Use read-only tools")
|
|
393
395
|
.option("--steer", "Interrupt the current turn and run immediately")
|
|
@@ -615,6 +617,7 @@ export function registerSpaces(program) {
|
|
|
615
617
|
.command("prompt [content...]", { hidden: true })
|
|
616
618
|
.alias("send")
|
|
617
619
|
.description("Send or schedule a prompt in the target space")
|
|
620
|
+
.option("--harness <harness>", "Harness: cohub, pi, or codex", "cohub")
|
|
618
621
|
.option("--session <id>", "Target session ID")
|
|
619
622
|
.option("--title <title>", "Title for a newly created session or schedule")
|
|
620
623
|
.option("-m, --model <model>", "Model name")
|
package/dist/index.js
CHANGED
|
@@ -17,7 +17,7 @@ import { registerReferences } from "./commands/references.js";
|
|
|
17
17
|
import { registerReferrals } from "./commands/referrals.js";
|
|
18
18
|
import { registerPrompt, registerSpaces } from "./commands/spaces.js";
|
|
19
19
|
import { maybeHandleRunCommand, printRunHelp } from "./commands/run.js";
|
|
20
|
-
import {
|
|
20
|
+
import { registerRuntime } from "./commands/runtime.js";
|
|
21
21
|
import { registerTasks } from "./commands/tasks.js";
|
|
22
22
|
import { registerDesktop, registerLegacyUi } from "./commands/desktop.js";
|
|
23
23
|
import { registerApps } from "./commands/apps.js";
|
|
@@ -53,7 +53,7 @@ Common commands:
|
|
|
53
53
|
cohub prompt "Fix the failing tests"
|
|
54
54
|
cohub completion "Summarize AGENTS.md" --system-prompt AGENTS.md --stream
|
|
55
55
|
cohub run -- git status
|
|
56
|
-
cohub
|
|
56
|
+
cohub runtime up ./my-project
|
|
57
57
|
cohub search "release notes"
|
|
58
58
|
cohub -s <space-id> boards inspect <board-id>
|
|
59
59
|
cohub -s <space-id> spaces turns ls --author others
|
|
@@ -82,7 +82,7 @@ registerProfile(program);
|
|
|
82
82
|
registerMe(program);
|
|
83
83
|
registerPrompt(program);
|
|
84
84
|
registerSpaces(program);
|
|
85
|
-
|
|
85
|
+
registerRuntime(program);
|
|
86
86
|
registerChannels(program);
|
|
87
87
|
registerGenerations(program);
|
|
88
88
|
registerModels(program);
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import { type HarnessArchive, type HarnessArchiveIndex, type RuntimeArchivePage, type RuntimeArchiveUpload } from "@neta-art/cohub";
|
|
2
|
+
export type ArchiveTransport = {
|
|
3
|
+
fetchObject?: typeof fetch;
|
|
4
|
+
prepareRuntimeArchive(index: HarnessArchiveIndex, options?: {
|
|
5
|
+
signal?: AbortSignal;
|
|
6
|
+
}): Promise<{
|
|
7
|
+
uploads: RuntimeArchiveUpload[];
|
|
8
|
+
}>;
|
|
9
|
+
commitRuntimeArchive(index: HarnessArchiveIndex, options?: {
|
|
10
|
+
signal?: AbortSignal;
|
|
11
|
+
}): Promise<{
|
|
12
|
+
ready: true;
|
|
13
|
+
}>;
|
|
14
|
+
getRuntimeArchive(sessionId: string, turnId: string, options?: {
|
|
15
|
+
signal?: AbortSignal;
|
|
16
|
+
}): Promise<RuntimeArchivePage>;
|
|
17
|
+
};
|
|
18
|
+
export declare function checksumNativeFile(path: string): Promise<string>;
|
|
19
|
+
export declare function atomicRuntimeJson(path: string, value: unknown): Promise<void>;
|
|
20
|
+
/** Immutable local segments are the outbox. Model acknowledgements never remove them. */
|
|
21
|
+
export declare class RuntimeArchiveStore {
|
|
22
|
+
readonly root: string;
|
|
23
|
+
private readonly transport?;
|
|
24
|
+
private flushing;
|
|
25
|
+
private readonly capturing;
|
|
26
|
+
constructor(root: string, transport?: ArchiveTransport | undefined);
|
|
27
|
+
pendingCount(): Promise<number>;
|
|
28
|
+
failedCaptureCount(): Promise<number>;
|
|
29
|
+
hasCapture(turnId: string): Promise<boolean>;
|
|
30
|
+
private version;
|
|
31
|
+
private blob;
|
|
32
|
+
private saveBlob;
|
|
33
|
+
private readIndex;
|
|
34
|
+
stage(state: {
|
|
35
|
+
sessionId: string;
|
|
36
|
+
harness: "pi" | "codex";
|
|
37
|
+
nativeSessionId: string;
|
|
38
|
+
path: string;
|
|
39
|
+
}, turnId: string): Promise<HarnessArchive>;
|
|
40
|
+
private capture;
|
|
41
|
+
flush(signal: AbortSignal): Promise<void>;
|
|
42
|
+
private drain;
|
|
43
|
+
restore(reference: HarnessArchive, target: string, signal?: AbortSignal): Promise<HarnessArchiveIndex>;
|
|
44
|
+
}
|
|
@@ -0,0 +1,353 @@
|
|
|
1
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
2
|
+
import { createReadStream } from "node:fs";
|
|
3
|
+
import { mkdir, open, readFile, readdir, rename, rm, stat, link } from "node:fs/promises";
|
|
4
|
+
import { dirname, join } from "node:path";
|
|
5
|
+
import { RUNTIME_ARCHIVE_SEGMENT_BYTES, harnessArchiveIndexSchema, validateArchiveBoundary, } from "@neta-art/cohub";
|
|
6
|
+
const missing = (error) => error?.code === "ENOENT";
|
|
7
|
+
const hash = (bytes, algorithm = "sha256") => createHash(algorithm).update(bytes).digest("hex");
|
|
8
|
+
export async function checksumNativeFile(path) {
|
|
9
|
+
const digest = createHash("sha256");
|
|
10
|
+
for await (const bytes of createReadStream(path))
|
|
11
|
+
digest.update(bytes);
|
|
12
|
+
return digest.digest("hex");
|
|
13
|
+
}
|
|
14
|
+
export async function atomicRuntimeJson(path, value) {
|
|
15
|
+
await mkdir(dirname(path), { recursive: true, mode: 0o700 });
|
|
16
|
+
const temporary = `${path}.${randomUUID()}.tmp`;
|
|
17
|
+
try {
|
|
18
|
+
const file = await open(temporary, "wx", 0o600);
|
|
19
|
+
try {
|
|
20
|
+
await file.writeFile(JSON.stringify(value));
|
|
21
|
+
await file.sync();
|
|
22
|
+
}
|
|
23
|
+
finally {
|
|
24
|
+
await file.close();
|
|
25
|
+
}
|
|
26
|
+
await rename(temporary, path);
|
|
27
|
+
if (process.platform !== "win32") {
|
|
28
|
+
const directory = await open(dirname(path), "r");
|
|
29
|
+
try {
|
|
30
|
+
await directory.sync();
|
|
31
|
+
}
|
|
32
|
+
finally {
|
|
33
|
+
await directory.close();
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
finally {
|
|
38
|
+
await rm(temporary, { force: true });
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
/** Immutable local segments are the outbox. Model acknowledgements never remove them. */
|
|
42
|
+
export class RuntimeArchiveStore {
|
|
43
|
+
root;
|
|
44
|
+
transport;
|
|
45
|
+
flushing = null;
|
|
46
|
+
capturing = new Map();
|
|
47
|
+
constructor(root, transport) {
|
|
48
|
+
this.root = root;
|
|
49
|
+
this.transport = transport;
|
|
50
|
+
}
|
|
51
|
+
async pendingCount() {
|
|
52
|
+
const pending = new Set();
|
|
53
|
+
for (const directory of ["pending", "captures"]) {
|
|
54
|
+
const names = await readdir(join(this.root, directory)).catch((error) => { if (missing(error))
|
|
55
|
+
return []; throw error; });
|
|
56
|
+
for (const name of names)
|
|
57
|
+
if (name.endsWith(".json"))
|
|
58
|
+
pending.add(name);
|
|
59
|
+
}
|
|
60
|
+
return pending.size;
|
|
61
|
+
}
|
|
62
|
+
async failedCaptureCount() {
|
|
63
|
+
const names = await readdir(join(this.root, "failed", "captures")).catch((error) => { if (missing(error))
|
|
64
|
+
return []; throw error; });
|
|
65
|
+
return names.filter((name) => name.endsWith(".json")).length;
|
|
66
|
+
}
|
|
67
|
+
async hasCapture(turnId) { return Boolean(await this.readIndex(this.version(turnId))); }
|
|
68
|
+
version(turnId) { return join(this.root, "versions", `${turnId}.json`); }
|
|
69
|
+
blob(sha) { return join(this.root, "objects", sha); }
|
|
70
|
+
async saveBlob(sha, bytes) {
|
|
71
|
+
await mkdir(join(this.root, "objects"), { recursive: true, mode: 0o700 });
|
|
72
|
+
const target = this.blob(sha), temporary = `${target}.${randomUUID()}.tmp`;
|
|
73
|
+
try {
|
|
74
|
+
const output = await open(temporary, "wx", 0o600);
|
|
75
|
+
try {
|
|
76
|
+
await output.writeFile(bytes);
|
|
77
|
+
await output.sync();
|
|
78
|
+
}
|
|
79
|
+
finally {
|
|
80
|
+
await output.close();
|
|
81
|
+
}
|
|
82
|
+
await link(temporary, target).catch((error) => { if (error.code !== "EEXIST")
|
|
83
|
+
throw error; });
|
|
84
|
+
}
|
|
85
|
+
finally {
|
|
86
|
+
await rm(temporary, { force: true });
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
async readIndex(path) {
|
|
90
|
+
try {
|
|
91
|
+
return harnessArchiveIndexSchema.parse(JSON.parse(await readFile(path, "utf8")));
|
|
92
|
+
}
|
|
93
|
+
catch (error) {
|
|
94
|
+
if (missing(error))
|
|
95
|
+
return null;
|
|
96
|
+
throw error;
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
stage(state, turnId) {
|
|
100
|
+
const existing = this.capturing.get(turnId);
|
|
101
|
+
if (existing)
|
|
102
|
+
return existing;
|
|
103
|
+
const task = this.capture(state, turnId).finally(() => this.capturing.delete(turnId));
|
|
104
|
+
this.capturing.set(turnId, task);
|
|
105
|
+
return task;
|
|
106
|
+
}
|
|
107
|
+
async capture(state, turnId) {
|
|
108
|
+
const identity = { sessionId: state.sessionId, turnId, harness: state.harness };
|
|
109
|
+
const headPath = join(this.root, "heads", `${state.sessionId}.${state.harness}.json`);
|
|
110
|
+
const saved = await this.readIndex(this.version(turnId));
|
|
111
|
+
if (saved) {
|
|
112
|
+
if (saved.sessionId !== state.sessionId || saved.harness !== state.harness)
|
|
113
|
+
throw new Error("Archive identity mismatch");
|
|
114
|
+
const committed = await stat(join(this.root, "ready", `${turnId}.json`)).catch((error) => { if (missing(error))
|
|
115
|
+
return null; throw error; });
|
|
116
|
+
if (!committed)
|
|
117
|
+
await atomicRuntimeJson(join(this.root, "pending", `${turnId}.json`), saved);
|
|
118
|
+
if (!await this.readIndex(headPath))
|
|
119
|
+
await atomicRuntimeJson(headPath, saved);
|
|
120
|
+
return identity;
|
|
121
|
+
}
|
|
122
|
+
const previous = await this.readIndex(headPath);
|
|
123
|
+
const file = await open(state.path, "r");
|
|
124
|
+
let index;
|
|
125
|
+
try {
|
|
126
|
+
const before = await file.stat();
|
|
127
|
+
if (!before.isFile() || !before.size)
|
|
128
|
+
throw new Error("Native archive is empty");
|
|
129
|
+
const buffer = Buffer.alloc(RUNTIME_ARCHIVE_SEGMENT_BYTES);
|
|
130
|
+
let offset = 0;
|
|
131
|
+
let digest = createHash("sha256");
|
|
132
|
+
let parent = null;
|
|
133
|
+
// Hash the old prefix, not just its size: equal-size and growing rewrites are valid.
|
|
134
|
+
if (previous && previous.nativeSessionId === state.nativeSessionId && previous.sizeBytes <= before.size) {
|
|
135
|
+
while (offset < previous.sizeBytes) {
|
|
136
|
+
const { bytesRead } = await file.read(buffer, 0, Math.min(buffer.length, previous.sizeBytes - offset), offset);
|
|
137
|
+
if (!bytesRead)
|
|
138
|
+
throw new Error("Native file changed during capture");
|
|
139
|
+
digest.update(buffer.subarray(0, bytesRead));
|
|
140
|
+
offset += bytesRead;
|
|
141
|
+
}
|
|
142
|
+
if (digest.copy().digest("hex") === previous.sha256)
|
|
143
|
+
parent = previous;
|
|
144
|
+
}
|
|
145
|
+
if (!parent) {
|
|
146
|
+
offset = 0;
|
|
147
|
+
digest = createHash("sha256");
|
|
148
|
+
}
|
|
149
|
+
const segments = [];
|
|
150
|
+
await mkdir(join(this.root, "objects"), { recursive: true, mode: 0o700 });
|
|
151
|
+
while (offset < before.size) {
|
|
152
|
+
const { bytesRead } = await file.read(buffer, 0, Math.min(buffer.length, before.size - offset), offset);
|
|
153
|
+
if (!bytesRead)
|
|
154
|
+
throw new Error("Native file changed during capture");
|
|
155
|
+
const bytes = buffer.subarray(0, bytesRead);
|
|
156
|
+
digest.update(bytes);
|
|
157
|
+
const segment = { offset, sizeBytes: bytesRead, sha256: hash(bytes), md5: hash(bytes, "md5") };
|
|
158
|
+
await this.saveBlob(segment.sha256, bytes);
|
|
159
|
+
segments.push(segment);
|
|
160
|
+
offset += bytesRead;
|
|
161
|
+
}
|
|
162
|
+
const after = await stat(state.path);
|
|
163
|
+
if (before.ino !== after.ino || before.size !== after.size || before.mtimeMs !== after.mtimeMs)
|
|
164
|
+
throw new Error("Native file changed during capture");
|
|
165
|
+
if (process.platform !== "win32") {
|
|
166
|
+
const directory = await open(join(this.root, "objects"), "r");
|
|
167
|
+
try {
|
|
168
|
+
await directory.sync();
|
|
169
|
+
}
|
|
170
|
+
finally {
|
|
171
|
+
await directory.close();
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
index = harnessArchiveIndexSchema.parse({ ...identity, version: 1, nativeSessionId: state.nativeSessionId,
|
|
175
|
+
nativeFormat: state.harness === "pi" ? "pi.jsonl" : "codex.rollout", parentTurnId: parent?.turnId ?? null,
|
|
176
|
+
sizeBytes: before.size, sha256: digest.digest("hex"), segments });
|
|
177
|
+
validateArchiveBoundary(index, parent);
|
|
178
|
+
}
|
|
179
|
+
finally {
|
|
180
|
+
await file.close();
|
|
181
|
+
}
|
|
182
|
+
// Publish the outbox before advancing the local head. Neither points at mutable files.
|
|
183
|
+
await atomicRuntimeJson(join(this.root, "pending", `${turnId}.json`), index);
|
|
184
|
+
await atomicRuntimeJson(this.version(turnId), index);
|
|
185
|
+
await atomicRuntimeJson(headPath, index);
|
|
186
|
+
return identity;
|
|
187
|
+
}
|
|
188
|
+
async flush(signal) {
|
|
189
|
+
if (!this.transport)
|
|
190
|
+
return;
|
|
191
|
+
if (this.flushing)
|
|
192
|
+
return this.flushing;
|
|
193
|
+
this.flushing = this.drain(signal).finally(() => { this.flushing = null; });
|
|
194
|
+
return this.flushing;
|
|
195
|
+
}
|
|
196
|
+
async drain(signal) {
|
|
197
|
+
const transport = this.transport;
|
|
198
|
+
if (!transport)
|
|
199
|
+
return;
|
|
200
|
+
const names = await readdir(join(this.root, "pending")).catch((error) => { if (missing(error))
|
|
201
|
+
return []; throw error; });
|
|
202
|
+
const pending = new Map();
|
|
203
|
+
for (const name of names) {
|
|
204
|
+
if (!name.endsWith(".json"))
|
|
205
|
+
continue;
|
|
206
|
+
const index = await this.readIndex(join(this.root, "pending", name));
|
|
207
|
+
if (index)
|
|
208
|
+
pending.set(index.turnId, index);
|
|
209
|
+
}
|
|
210
|
+
const children = new Map();
|
|
211
|
+
const queue = [];
|
|
212
|
+
for (const index of pending.values()) {
|
|
213
|
+
if (index.parentTurnId && pending.has(index.parentTurnId)) {
|
|
214
|
+
const siblings = children.get(index.parentTurnId) ?? [];
|
|
215
|
+
siblings.push(index);
|
|
216
|
+
children.set(index.parentTurnId, siblings);
|
|
217
|
+
}
|
|
218
|
+
else
|
|
219
|
+
queue.push(index);
|
|
220
|
+
}
|
|
221
|
+
if (pending.size && !queue.length)
|
|
222
|
+
throw new Error("Cyclic archive outbox");
|
|
223
|
+
for (let cursor = 0; cursor < queue.length; cursor++) {
|
|
224
|
+
signal.throwIfAborted();
|
|
225
|
+
const index = queue[cursor];
|
|
226
|
+
if (!index)
|
|
227
|
+
continue;
|
|
228
|
+
try {
|
|
229
|
+
const { uploads } = await transport.prepareRuntimeArchive(index, { signal: AbortSignal.any([signal, AbortSignal.timeout(30_000)]) });
|
|
230
|
+
if (uploads.length > index.segments.length)
|
|
231
|
+
throw new Error("Upload plan mismatch");
|
|
232
|
+
const expected = new Set(index.segments.map((segment) => JSON.stringify(segment)));
|
|
233
|
+
for (const { segment, uploadUrl, headers } of uploads) {
|
|
234
|
+
if (!expected.has(JSON.stringify(segment)))
|
|
235
|
+
throw new Error("Upload segment mismatch");
|
|
236
|
+
signal.throwIfAborted();
|
|
237
|
+
const bytes = await readFile(this.blob(segment.sha256));
|
|
238
|
+
if (bytes.length !== segment.sizeBytes || hash(bytes) !== segment.sha256)
|
|
239
|
+
throw new Error("Local archive segment is corrupt");
|
|
240
|
+
const response = await (transport.fetchObject ?? fetch)(uploadUrl, { method: "PUT", headers, body: bytes, redirect: "error", signal: AbortSignal.any([signal, AbortSignal.timeout(60_000)]) });
|
|
241
|
+
// An immutable segment may already exist after a lost acknowledgement. Commit verifies it.
|
|
242
|
+
if (!response.ok && ![409, 412].includes(response.status))
|
|
243
|
+
throw new Error(`Archive upload failed: ${response.status}`);
|
|
244
|
+
}
|
|
245
|
+
await transport.commitRuntimeArchive(index, { signal: AbortSignal.any([signal, AbortSignal.timeout(30_000)]) });
|
|
246
|
+
await atomicRuntimeJson(join(this.root, "ready", `${index.turnId}.json`), { turnId: index.turnId, sha256: index.sha256 });
|
|
247
|
+
await rm(join(this.root, "pending", `${index.turnId}.json`), { force: true });
|
|
248
|
+
queue.push(...children.get(index.turnId) ?? []);
|
|
249
|
+
}
|
|
250
|
+
catch (error) {
|
|
251
|
+
if (!signal.aborted)
|
|
252
|
+
console.error("Archive pending; native segments retained:", error);
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
async restore(reference, target, signal) {
|
|
257
|
+
if (!this.transport)
|
|
258
|
+
throw new Error("Archive transport unavailable");
|
|
259
|
+
const timeout = (ms) => signal ? AbortSignal.any([signal, AbortSignal.timeout(ms)]) : AbortSignal.timeout(ms);
|
|
260
|
+
const pages = [];
|
|
261
|
+
const visited = new Set();
|
|
262
|
+
let turnId = reference.turnId;
|
|
263
|
+
while (turnId) {
|
|
264
|
+
signal?.throwIfAborted();
|
|
265
|
+
if (visited.has(turnId))
|
|
266
|
+
throw new Error("Cyclic archive");
|
|
267
|
+
visited.add(turnId);
|
|
268
|
+
const page = await this.transport.getRuntimeArchive(reference.sessionId, turnId, { signal: timeout(30_000) });
|
|
269
|
+
const index = harnessArchiveIndexSchema.parse(page.index);
|
|
270
|
+
if (index.turnId !== turnId || index.sessionId !== reference.sessionId || index.harness !== reference.harness)
|
|
271
|
+
throw new Error("Archive identity mismatch");
|
|
272
|
+
pages.push({ ...page, index });
|
|
273
|
+
turnId = index.parentTurnId;
|
|
274
|
+
}
|
|
275
|
+
const head = pages[0]?.index;
|
|
276
|
+
if (!head)
|
|
277
|
+
throw new Error("Archive missing");
|
|
278
|
+
await mkdir(dirname(target), { recursive: true, mode: 0o700 });
|
|
279
|
+
const temporary = `${target}.${randomUUID()}.restoring`;
|
|
280
|
+
const file = await open(temporary, "wx", 0o600);
|
|
281
|
+
try {
|
|
282
|
+
let parent = null;
|
|
283
|
+
const digest = createHash("sha256");
|
|
284
|
+
for (const page of pages.reverse()) {
|
|
285
|
+
validateArchiveBoundary(page.index, parent);
|
|
286
|
+
if (page.segments.length !== page.index.segments.length)
|
|
287
|
+
throw new Error("Missing archive segments");
|
|
288
|
+
for (const [ordinal, expected] of page.index.segments.entries()) {
|
|
289
|
+
signal?.throwIfAborted();
|
|
290
|
+
const cached = await readFile(this.blob(expected.sha256)).catch((error) => { if (missing(error))
|
|
291
|
+
return null; throw error; });
|
|
292
|
+
if (cached) {
|
|
293
|
+
if (cached.length !== expected.sizeBytes || hash(cached) !== expected.sha256)
|
|
294
|
+
throw new Error("Cached archive segment is corrupt");
|
|
295
|
+
digest.update(cached);
|
|
296
|
+
await file.writeFile(cached);
|
|
297
|
+
continue;
|
|
298
|
+
}
|
|
299
|
+
let link = page.segments[ordinal];
|
|
300
|
+
if (!link || JSON.stringify(link.segment) !== JSON.stringify(expected))
|
|
301
|
+
throw new Error("Archive segment identity mismatch");
|
|
302
|
+
let response = await (this.transport.fetchObject ?? fetch)(link.downloadUrl, { redirect: "error", signal: timeout(60_000) });
|
|
303
|
+
if ([401, 403].includes(response.status)) {
|
|
304
|
+
const refreshed = await this.transport.getRuntimeArchive(reference.sessionId, page.index.turnId, { signal: timeout(30_000) });
|
|
305
|
+
link = refreshed.segments[ordinal];
|
|
306
|
+
if (!link || JSON.stringify(link.segment) !== JSON.stringify(expected))
|
|
307
|
+
throw new Error("Archive segment missing");
|
|
308
|
+
response = await (this.transport.fetchObject ?? fetch)(link.downloadUrl, { redirect: "error", signal: timeout(60_000) });
|
|
309
|
+
}
|
|
310
|
+
if (!response.ok || !response.body)
|
|
311
|
+
throw new Error(`Archive download failed: ${response.status}`);
|
|
312
|
+
const segmentHash = createHash("sha256");
|
|
313
|
+
let size = 0;
|
|
314
|
+
const chunks = [];
|
|
315
|
+
for await (const chunk of response.body) {
|
|
316
|
+
size += chunk.length;
|
|
317
|
+
if (size > expected.sizeBytes)
|
|
318
|
+
throw new Error("Archive size mismatch");
|
|
319
|
+
chunks.push(chunk);
|
|
320
|
+
segmentHash.update(chunk);
|
|
321
|
+
digest.update(chunk);
|
|
322
|
+
await file.writeFile(chunk);
|
|
323
|
+
}
|
|
324
|
+
if (size !== expected.sizeBytes || segmentHash.digest("hex") !== expected.sha256)
|
|
325
|
+
throw new Error("Archive checksum mismatch");
|
|
326
|
+
await this.saveBlob(expected.sha256, Buffer.concat(chunks));
|
|
327
|
+
}
|
|
328
|
+
if (digest.copy().digest("hex") !== page.index.sha256)
|
|
329
|
+
throw new Error("Archive version checksum mismatch");
|
|
330
|
+
parent = page.index;
|
|
331
|
+
}
|
|
332
|
+
if ((await file.stat()).size !== head.sizeBytes)
|
|
333
|
+
throw new Error("Archive length mismatch");
|
|
334
|
+
await file.sync();
|
|
335
|
+
await file.close();
|
|
336
|
+
await link(temporary, target);
|
|
337
|
+
if (process.platform !== "win32") {
|
|
338
|
+
const directory = await open(dirname(target), "r");
|
|
339
|
+
try {
|
|
340
|
+
await directory.sync();
|
|
341
|
+
}
|
|
342
|
+
finally {
|
|
343
|
+
await directory.close();
|
|
344
|
+
}
|
|
345
|
+
}
|
|
346
|
+
return head;
|
|
347
|
+
}
|
|
348
|
+
finally {
|
|
349
|
+
await file.close();
|
|
350
|
+
await rm(temporary, { force: true });
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { RuntimeMessage } from "@neta-art/cohub";
|
|
2
|
+
declare const fields: readonly ["inputTokens", "outputTokens", "cachedInputTokens", "cacheWriteInputTokens", "totalTokens"];
|
|
3
|
+
export type CodexTokenTotals = Record<typeof fields[number], number>;
|
|
4
|
+
export declare const codexTokenTotals: (value: unknown) => CodexTokenTotals;
|
|
5
|
+
export declare const subtractCodexTokens: (total: CodexTokenTotals, base: CodexTokenTotals) => CodexTokenTotals;
|
|
6
|
+
export declare function codexUsage(total: CodexTokenTotals): NonNullable<RuntimeMessage["usage"]>;
|
|
7
|
+
/** Seed portable imports from the original native counters, not a prior turn's `last`. */
|
|
8
|
+
export declare function codexArchiveTotals(records: Record<string, unknown>[]): CodexTokenTotals | undefined;
|
|
9
|
+
export {};
|