@phnx-labs/agents-cli 1.20.15 → 1.20.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +9 -0
- package/dist/commands/secrets.js +53 -1
- package/dist/commands/sessions-sync.d.ts +13 -0
- package/dist/commands/sessions-sync.js +73 -0
- package/dist/commands/sessions.js +2 -0
- package/dist/commands/sync.d.ts +10 -3
- package/dist/commands/sync.js +72 -9
- package/dist/commands/view.js +11 -3
- package/dist/index.js +1 -1
- package/dist/lib/agents.d.ts +11 -0
- package/dist/lib/agents.js +11 -9
- package/dist/lib/daemon.d.ts +19 -0
- package/dist/lib/daemon.js +97 -2
- package/dist/lib/hooks.js +12 -0
- package/dist/lib/migrate.d.ts +22 -0
- package/dist/lib/migrate.js +99 -1
- package/dist/lib/plugin-marketplace.d.ts +15 -0
- package/dist/lib/plugin-marketplace.js +54 -0
- package/dist/lib/secrets/drivers/rush.d.ts +14 -0
- package/dist/lib/secrets/drivers/rush.js +84 -0
- package/dist/lib/secrets/index.js +20 -0
- package/dist/lib/secrets/linux.js +88 -10
- package/dist/lib/secrets/sync-backend.d.ts +48 -0
- package/dist/lib/secrets/sync-backend.js +13 -0
- package/dist/lib/secrets/sync.d.ts +15 -23
- package/dist/lib/secrets/sync.js +31 -66
- package/dist/lib/session/parse.d.ts +2 -0
- package/dist/lib/session/parse.js +168 -2
- package/dist/lib/session/sync/agents.d.ts +46 -0
- package/dist/lib/session/sync/agents.js +94 -0
- package/dist/lib/session/sync/config.d.ts +30 -0
- package/dist/lib/session/sync/config.js +58 -0
- package/dist/lib/session/sync/crdt.d.ts +44 -0
- package/dist/lib/session/sync/crdt.js +119 -0
- package/dist/lib/session/sync/manifest.d.ts +51 -0
- package/dist/lib/session/sync/manifest.js +96 -0
- package/dist/lib/session/sync/r2.d.ts +32 -0
- package/dist/lib/session/sync/r2.js +121 -0
- package/dist/lib/session/sync/sync.d.ts +82 -0
- package/dist/lib/session/sync/sync.js +251 -0
- package/dist/lib/shims.d.ts +1 -1
- package/dist/lib/shims.js +17 -1
- package/dist/lib/sync-umbrella.d.ts +76 -0
- package/dist/lib/sync-umbrella.js +125 -0
- package/dist/lib/teams/parsers.js +159 -1
- package/dist/lib/usage.d.ts +18 -0
- package/dist/lib/usage.js +25 -0
- package/dist/lib/versions.js +30 -13
- package/package.json +2 -1
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Per-agent adapter for sync. Each supported agent declares where its
|
|
3
|
+
* transcripts live and how to derive a session id and storage-relative key
|
|
4
|
+
* from a file path. The merge (crdt.ts) and transport (r2.ts) are fully
|
|
5
|
+
* agent-agnostic — adding a new agent is just another entry in SYNC_AGENTS.
|
|
6
|
+
*
|
|
7
|
+
* Mirror layout: synced-in transcripts land under
|
|
8
|
+
* ~/.agents/.history/backups/<agent>/<machine>/<subdir>/<relKey>
|
|
9
|
+
* which is already a scan root (getAgentSessionDirs scans backups/<agent>/<ts>),
|
|
10
|
+
* so the existing incremental scanner indexes them with no changes. Because the
|
|
11
|
+
* scanner dedups by session id with the live home scanned first, a session that
|
|
12
|
+
* also exists locally always wins — the mirror only ever fills in sessions
|
|
13
|
+
* originated on other machines.
|
|
14
|
+
*/
|
|
15
|
+
import * as fs from 'fs';
|
|
16
|
+
import * as path from 'path';
|
|
17
|
+
import { getHistoryDir } from '../../state.js';
|
|
18
|
+
import { getAgentSessionDirs } from '../discover.js';
|
|
19
|
+
import { walkForFiles } from '../../fs-walk.js';
|
|
20
|
+
const UUID_RE = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/i;
|
|
21
|
+
export const SYNC_AGENTS = [
|
|
22
|
+
{
|
|
23
|
+
id: 'claude',
|
|
24
|
+
subdir: 'projects',
|
|
25
|
+
// Claude transcripts are <projectDir>/<sessionId>.jsonl.
|
|
26
|
+
sessionIdFromRelKey: rel => path.basename(rel).replace(/\.jsonl$/, ''),
|
|
27
|
+
},
|
|
28
|
+
{
|
|
29
|
+
id: 'codex',
|
|
30
|
+
subdir: 'sessions',
|
|
31
|
+
// Codex transcripts are rollout-<ts>-<uuid>.jsonl under date dirs; the uuid
|
|
32
|
+
// is the session id (matches session_meta.payload.id).
|
|
33
|
+
sessionIdFromRelKey: rel => path.basename(rel).match(UUID_RE)?.[0] ?? rel,
|
|
34
|
+
},
|
|
35
|
+
];
|
|
36
|
+
let cachedMirrorRoot = null;
|
|
37
|
+
function mirrorRootReal() {
|
|
38
|
+
if (cachedMirrorRoot)
|
|
39
|
+
return cachedMirrorRoot;
|
|
40
|
+
const root = path.join(getHistoryDir(), 'backups');
|
|
41
|
+
cachedMirrorRoot = safeReal(root);
|
|
42
|
+
return cachedMirrorRoot;
|
|
43
|
+
}
|
|
44
|
+
function safeReal(p) {
|
|
45
|
+
try {
|
|
46
|
+
return fs.realpathSync(p);
|
|
47
|
+
}
|
|
48
|
+
catch {
|
|
49
|
+
return path.resolve(p);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* List this machine's own transcript files for an agent, EXCLUDING the sync
|
|
54
|
+
* mirror (we never re-upload another machine's files under our prefix). Dedups
|
|
55
|
+
* by session id so a session present in multiple version homes is uploaded once.
|
|
56
|
+
*/
|
|
57
|
+
export function listLocalTranscripts(spec) {
|
|
58
|
+
const mirror = mirrorRootReal();
|
|
59
|
+
const out = [];
|
|
60
|
+
const seen = new Set();
|
|
61
|
+
for (const dir of getAgentSessionDirs(spec.id, spec.subdir)) {
|
|
62
|
+
if (safeReal(dir).startsWith(mirror))
|
|
63
|
+
continue; // skip synced-in mirror dirs
|
|
64
|
+
for (const abs of walkForFiles(dir, '.jsonl', 100_000)) {
|
|
65
|
+
const relKey = path.relative(dir, abs);
|
|
66
|
+
if (!relKey || relKey.startsWith('..'))
|
|
67
|
+
continue;
|
|
68
|
+
const sessionId = spec.sessionIdFromRelKey(relKey);
|
|
69
|
+
if (seen.has(sessionId))
|
|
70
|
+
continue;
|
|
71
|
+
seen.add(sessionId);
|
|
72
|
+
out.push({ absPath: abs, sessionId, relKey });
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
return out;
|
|
76
|
+
}
|
|
77
|
+
/** Session ids this machine holds locally (live home), used to skip mirror writes. */
|
|
78
|
+
export function localSessionIds(spec) {
|
|
79
|
+
return new Set(listLocalTranscripts(spec).map(t => t.sessionId));
|
|
80
|
+
}
|
|
81
|
+
/** Absolute mirror path for a remote machine's transcript — lands in a scan root. */
|
|
82
|
+
export function mirrorPath(spec, machine, relKey) {
|
|
83
|
+
return path.join(getHistoryDir(), 'backups', spec.id, machine, spec.subdir, relKey);
|
|
84
|
+
}
|
|
85
|
+
/** R2 object key for a transcript: sessions/<machine>/<agent>/<sessionId>.jsonl */
|
|
86
|
+
export function objectKey(machine, agentId, sessionId) {
|
|
87
|
+
return `sessions/${machine}/${agentId}/${sessionId}.jsonl`;
|
|
88
|
+
}
|
|
89
|
+
/** R2 object key for a machine's manifest. */
|
|
90
|
+
export function manifestKey(machine) {
|
|
91
|
+
return `sessions/${machine}/manifest.json`;
|
|
92
|
+
}
|
|
93
|
+
/** Prefix under which all machine manifests live (for discovery). */
|
|
94
|
+
export const SESSIONS_PREFIX = 'sessions/';
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Configuration for cross-machine session sync: R2 credentials and this
|
|
3
|
+
* machine's stable identity. Credentials come from the `r2.backups` secrets
|
|
4
|
+
* bundle (OS keychain on macOS, libsecret on Linux) — never from env or disk.
|
|
5
|
+
*/
|
|
6
|
+
/** Secrets bundle holding the R2 credentials. */
|
|
7
|
+
export declare const SYNC_BUNDLE = "r2.backups";
|
|
8
|
+
export interface R2Config {
|
|
9
|
+
accountId: string;
|
|
10
|
+
bucket: string;
|
|
11
|
+
accessKeyId: string;
|
|
12
|
+
secretAccessKey: string;
|
|
13
|
+
/** S3-compatible endpoint for the account (no bucket, no trailing slash). */
|
|
14
|
+
endpoint: string;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Resolve R2 credentials from the `r2.backups` bundle. Throws a clear,
|
|
18
|
+
* actionable error if the bundle or any key is missing — sync cannot proceed
|
|
19
|
+
* without real credentials (no silent fallback).
|
|
20
|
+
*/
|
|
21
|
+
export declare function loadR2Config(): R2Config;
|
|
22
|
+
/** True when the sync bundle exists and looks resolvable, without throwing. */
|
|
23
|
+
export declare function isSyncConfigured(): boolean;
|
|
24
|
+
/**
|
|
25
|
+
* This machine's stable, human-readable id, used as its R2 prefix and mirror
|
|
26
|
+
* directory name. Tailnet hostnames (zion, yosemite-s0, mac-mini) are already
|
|
27
|
+
* unique and readable; we lowercase and strip any domain suffix. Overridable
|
|
28
|
+
* via AGENTS_SYNC_MACHINE_ID for tests and unusual setups.
|
|
29
|
+
*/
|
|
30
|
+
export declare function machineId(): string;
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Configuration for cross-machine session sync: R2 credentials and this
|
|
3
|
+
* machine's stable identity. Credentials come from the `r2.backups` secrets
|
|
4
|
+
* bundle (OS keychain on macOS, libsecret on Linux) — never from env or disk.
|
|
5
|
+
*/
|
|
6
|
+
import * as os from 'os';
|
|
7
|
+
import { readAndResolveBundleEnv } from '../../secrets/bundles.js';
|
|
8
|
+
/** Secrets bundle holding the R2 credentials. */
|
|
9
|
+
export const SYNC_BUNDLE = 'r2.backups';
|
|
10
|
+
/**
|
|
11
|
+
* Resolve R2 credentials from the `r2.backups` bundle. Throws a clear,
|
|
12
|
+
* actionable error if the bundle or any key is missing — sync cannot proceed
|
|
13
|
+
* without real credentials (no silent fallback).
|
|
14
|
+
*/
|
|
15
|
+
export function loadR2Config() {
|
|
16
|
+
const { env } = readAndResolveBundleEnv(SYNC_BUNDLE, { caller: 'sessions-sync' });
|
|
17
|
+
const accountId = env.R2_ACCOUNT_ID?.trim();
|
|
18
|
+
const bucket = env.R2_BUCKET_NAME?.trim();
|
|
19
|
+
const accessKeyId = env.R2_ACCESS_KEY_ID?.trim();
|
|
20
|
+
const secretAccessKey = env.R2_SECRET_ACCESS_KEY?.trim();
|
|
21
|
+
const missing = [
|
|
22
|
+
!accountId && 'R2_ACCOUNT_ID',
|
|
23
|
+
!bucket && 'R2_BUCKET_NAME',
|
|
24
|
+
!accessKeyId && 'R2_ACCESS_KEY_ID',
|
|
25
|
+
!secretAccessKey && 'R2_SECRET_ACCESS_KEY',
|
|
26
|
+
].filter(Boolean);
|
|
27
|
+
if (missing.length > 0) {
|
|
28
|
+
throw new Error(`Sessions sync: bundle '${SYNC_BUNDLE}' is missing ${missing.join(', ')}. ` +
|
|
29
|
+
`Add them with: agents secrets add ${SYNC_BUNDLE} <KEY>`);
|
|
30
|
+
}
|
|
31
|
+
return {
|
|
32
|
+
accountId: accountId,
|
|
33
|
+
bucket: bucket,
|
|
34
|
+
accessKeyId: accessKeyId,
|
|
35
|
+
secretAccessKey: secretAccessKey,
|
|
36
|
+
endpoint: `https://${accountId}.r2.cloudflarestorage.com`,
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
/** True when the sync bundle exists and looks resolvable, without throwing. */
|
|
40
|
+
export function isSyncConfigured() {
|
|
41
|
+
try {
|
|
42
|
+
loadR2Config();
|
|
43
|
+
return true;
|
|
44
|
+
}
|
|
45
|
+
catch {
|
|
46
|
+
return false;
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* This machine's stable, human-readable id, used as its R2 prefix and mirror
|
|
51
|
+
* directory name. Tailnet hostnames (zion, yosemite-s0, mac-mini) are already
|
|
52
|
+
* unique and readable; we lowercase and strip any domain suffix. Overridable
|
|
53
|
+
* via AGENTS_SYNC_MACHINE_ID for tests and unusual setups.
|
|
54
|
+
*/
|
|
55
|
+
export function machineId() {
|
|
56
|
+
const raw = process.env.AGENTS_SYNC_MACHINE_ID || os.hostname();
|
|
57
|
+
return raw.split('.')[0].trim().toLowerCase().replace(/[^a-z0-9_-]/g, '-') || 'unknown';
|
|
58
|
+
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* CRDT merge for agent transcripts.
|
|
3
|
+
*
|
|
4
|
+
* A transcript (Claude JSONL, Codex JSONL, …) is an append-only log of
|
|
5
|
+
* immutable events: each line is written once and never rewritten, and Claude
|
|
6
|
+
* already tolerates branches within one file (parentUuid fan-out). That makes a
|
|
7
|
+
* transcript a grow-only set (G-Set) of events, and merging two copies of the
|
|
8
|
+
* same session is a set union — associative, commutative, idempotent. Two
|
|
9
|
+
* machines that each appended to the same session therefore converge to the
|
|
10
|
+
* exact same merged file regardless of sync order or timing, with zero conflict
|
|
11
|
+
* resolution and zero data loss.
|
|
12
|
+
*
|
|
13
|
+
* Events are identified by the SHA-256 of their raw line bytes. We deliberately
|
|
14
|
+
* do NOT key on a per-event `uuid`: Codex lines carry no id, and because an
|
|
15
|
+
* event is written exactly once and then copied verbatim across machines, the
|
|
16
|
+
* raw bytes are a stable, agent-agnostic identity. Multiplicity is preserved
|
|
17
|
+
* (some transcripts contain legitimately identical lines, e.g. paired
|
|
18
|
+
* `queue-operation` entries) by taking the per-hash max count across sources.
|
|
19
|
+
*/
|
|
20
|
+
export interface ParsedEvent {
|
|
21
|
+
/** Original line bytes, exactly as stored (no trailing newline). */
|
|
22
|
+
raw: string;
|
|
23
|
+
/** SHA-256 of `raw` — the event's identity. */
|
|
24
|
+
hash: string;
|
|
25
|
+
/** Top-level ISO `timestamp`, or '' when absent/unparseable. */
|
|
26
|
+
ts: string;
|
|
27
|
+
}
|
|
28
|
+
/** Parse a transcript's raw text into events, skipping blank lines. */
|
|
29
|
+
export declare function parseTranscript(content: string): ParsedEvent[];
|
|
30
|
+
/**
|
|
31
|
+
* Merge copies of the same session into one transcript via G-Set union.
|
|
32
|
+
*
|
|
33
|
+
* Returns a source VERBATIM (no reordering, byte-identical) for the common
|
|
34
|
+
* cases — one source, all sources identical, or one source a superset of the
|
|
35
|
+
* rest — so the steady state never rewrites unchanged files. Only a true fork
|
|
36
|
+
* (each side holds events the other lacks) produces a reordered union, sorted
|
|
37
|
+
* by (timestamp, hash) so every machine derives identical bytes.
|
|
38
|
+
*/
|
|
39
|
+
export declare function mergeTranscripts(contents: string[]): string;
|
|
40
|
+
/** Count distinct + total events across copies (for logging / manifest stats). */
|
|
41
|
+
export declare function transcriptStats(content: string): {
|
|
42
|
+
events: number;
|
|
43
|
+
lastTs: string;
|
|
44
|
+
};
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* CRDT merge for agent transcripts.
|
|
3
|
+
*
|
|
4
|
+
* A transcript (Claude JSONL, Codex JSONL, …) is an append-only log of
|
|
5
|
+
* immutable events: each line is written once and never rewritten, and Claude
|
|
6
|
+
* already tolerates branches within one file (parentUuid fan-out). That makes a
|
|
7
|
+
* transcript a grow-only set (G-Set) of events, and merging two copies of the
|
|
8
|
+
* same session is a set union — associative, commutative, idempotent. Two
|
|
9
|
+
* machines that each appended to the same session therefore converge to the
|
|
10
|
+
* exact same merged file regardless of sync order or timing, with zero conflict
|
|
11
|
+
* resolution and zero data loss.
|
|
12
|
+
*
|
|
13
|
+
* Events are identified by the SHA-256 of their raw line bytes. We deliberately
|
|
14
|
+
* do NOT key on a per-event `uuid`: Codex lines carry no id, and because an
|
|
15
|
+
* event is written exactly once and then copied verbatim across machines, the
|
|
16
|
+
* raw bytes are a stable, agent-agnostic identity. Multiplicity is preserved
|
|
17
|
+
* (some transcripts contain legitimately identical lines, e.g. paired
|
|
18
|
+
* `queue-operation` entries) by taking the per-hash max count across sources.
|
|
19
|
+
*/
|
|
20
|
+
import * as crypto from 'crypto';
|
|
21
|
+
/** Extract the top-level `timestamp` field both Claude and Codex stamp per line. */
|
|
22
|
+
function lineTimestamp(raw) {
|
|
23
|
+
try {
|
|
24
|
+
const obj = JSON.parse(raw);
|
|
25
|
+
const ts = obj?.timestamp;
|
|
26
|
+
return typeof ts === 'string' ? ts : '';
|
|
27
|
+
}
|
|
28
|
+
catch {
|
|
29
|
+
return '';
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
/** Parse a transcript's raw text into events, skipping blank lines. */
|
|
33
|
+
export function parseTranscript(content) {
|
|
34
|
+
const out = [];
|
|
35
|
+
for (const line of content.split('\n')) {
|
|
36
|
+
if (line.trim() === '')
|
|
37
|
+
continue;
|
|
38
|
+
out.push({
|
|
39
|
+
raw: line,
|
|
40
|
+
hash: crypto.createHash('sha256').update(line).digest('hex'),
|
|
41
|
+
ts: lineTimestamp(line),
|
|
42
|
+
});
|
|
43
|
+
}
|
|
44
|
+
return out;
|
|
45
|
+
}
|
|
46
|
+
/** Order events deterministically across machines: by timestamp, then hash. */
|
|
47
|
+
function compareEvents(a, b) {
|
|
48
|
+
if (a.ts !== b.ts)
|
|
49
|
+
return a.ts < b.ts ? -1 : 1;
|
|
50
|
+
if (a.hash !== b.hash)
|
|
51
|
+
return a.hash < b.hash ? -1 : 1;
|
|
52
|
+
return 0;
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Merge copies of the same session into one transcript via G-Set union.
|
|
56
|
+
*
|
|
57
|
+
* Returns a source VERBATIM (no reordering, byte-identical) for the common
|
|
58
|
+
* cases — one source, all sources identical, or one source a superset of the
|
|
59
|
+
* rest — so the steady state never rewrites unchanged files. Only a true fork
|
|
60
|
+
* (each side holds events the other lacks) produces a reordered union, sorted
|
|
61
|
+
* by (timestamp, hash) so every machine derives identical bytes.
|
|
62
|
+
*/
|
|
63
|
+
export function mergeTranscripts(contents) {
|
|
64
|
+
const sources = contents.filter(c => c.length > 0);
|
|
65
|
+
if (sources.length === 0)
|
|
66
|
+
return '';
|
|
67
|
+
if (sources.length === 1)
|
|
68
|
+
return sources[0];
|
|
69
|
+
const parsed = sources.map(parseTranscript);
|
|
70
|
+
// Per-source multiset of event hashes.
|
|
71
|
+
const counts = parsed.map(events => {
|
|
72
|
+
const m = new Map();
|
|
73
|
+
for (const e of events)
|
|
74
|
+
m.set(e.hash, (m.get(e.hash) ?? 0) + 1);
|
|
75
|
+
return m;
|
|
76
|
+
});
|
|
77
|
+
// Global max count per hash + a representative event for raw bytes / ts.
|
|
78
|
+
const maxCount = new Map();
|
|
79
|
+
const rep = new Map();
|
|
80
|
+
for (let i = 0; i < parsed.length; i++) {
|
|
81
|
+
for (const e of parsed[i])
|
|
82
|
+
if (!rep.has(e.hash))
|
|
83
|
+
rep.set(e.hash, e);
|
|
84
|
+
for (const [h, c] of counts[i])
|
|
85
|
+
maxCount.set(h, Math.max(maxCount.get(h) ?? 0, c));
|
|
86
|
+
}
|
|
87
|
+
// Superset fast path: a source whose multiset already equals the global max
|
|
88
|
+
// is the union — return it byte-for-byte (covers identical/subset/prefix).
|
|
89
|
+
for (let i = 0; i < sources.length; i++) {
|
|
90
|
+
let isSuperset = true;
|
|
91
|
+
for (const [h, c] of maxCount) {
|
|
92
|
+
if ((counts[i].get(h) ?? 0) !== c) {
|
|
93
|
+
isSuperset = false;
|
|
94
|
+
break;
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
if (isSuperset)
|
|
98
|
+
return sources[i];
|
|
99
|
+
}
|
|
100
|
+
// True fork: emit each distinct event maxCount times, deterministically ordered.
|
|
101
|
+
const distinct = [...rep.values()].sort(compareEvents);
|
|
102
|
+
const lines = [];
|
|
103
|
+
for (const e of distinct) {
|
|
104
|
+
const n = maxCount.get(e.hash) ?? 1;
|
|
105
|
+
for (let k = 0; k < n; k++)
|
|
106
|
+
lines.push(e.raw);
|
|
107
|
+
}
|
|
108
|
+
const trailing = sources.some(s => s.endsWith('\n')) ? '\n' : '';
|
|
109
|
+
return lines.join('\n') + trailing;
|
|
110
|
+
}
|
|
111
|
+
/** Count distinct + total events across copies (for logging / manifest stats). */
|
|
112
|
+
export function transcriptStats(content) {
|
|
113
|
+
const parsed = parseTranscript(content);
|
|
114
|
+
let lastTs = '';
|
|
115
|
+
for (const e of parsed)
|
|
116
|
+
if (e.ts > lastTs)
|
|
117
|
+
lastTs = e.ts;
|
|
118
|
+
return { events: parsed.length, lastTs };
|
|
119
|
+
}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Manifests and the local upload ledger.
|
|
3
|
+
*
|
|
4
|
+
* Each machine publishes one manifest object (sessions/<machine>/manifest.json)
|
|
5
|
+
* listing every session it holds, keyed by session id, with content hash + size
|
|
6
|
+
* + last event timestamp. Pulling machines diff manifests to fetch only changed
|
|
7
|
+
* objects — no per-tick LIST of thousands of keys.
|
|
8
|
+
*
|
|
9
|
+
* The local upload ledger (cache, per machine) records (size, mtime, hash) of
|
|
10
|
+
* files already pushed, so a tick re-uploads only files that actually grew.
|
|
11
|
+
*/
|
|
12
|
+
export interface ManifestEntry {
|
|
13
|
+
/** Storage-relative key, preserved into the mirror layout on the puller. */
|
|
14
|
+
relKey: string;
|
|
15
|
+
size: number;
|
|
16
|
+
/** SHA-256 of the full transcript file. */
|
|
17
|
+
hash: string;
|
|
18
|
+
/** Latest event timestamp in the transcript. */
|
|
19
|
+
lastTs: string;
|
|
20
|
+
}
|
|
21
|
+
/** sessionId -> entry */
|
|
22
|
+
export type AgentManifest = Record<string, ManifestEntry>;
|
|
23
|
+
export interface Manifest {
|
|
24
|
+
machine: string;
|
|
25
|
+
updatedAt: string;
|
|
26
|
+
/** agentId -> (sessionId -> entry) */
|
|
27
|
+
agents: Record<string, AgentManifest>;
|
|
28
|
+
}
|
|
29
|
+
export declare function emptyManifest(machine: string, updatedAt: string): Manifest;
|
|
30
|
+
export declare function parseManifest(text: string): Manifest | null;
|
|
31
|
+
export declare function hashContent(content: string | Uint8Array): string;
|
|
32
|
+
export declare function loadLocalManifest(): Manifest | null;
|
|
33
|
+
export declare function saveLocalManifest(m: Manifest): void;
|
|
34
|
+
/** key `${agent}/${sessionId}` -> signature of applied source hashes */
|
|
35
|
+
export type PullState = Record<string, string>;
|
|
36
|
+
export declare function loadPullState(): PullState;
|
|
37
|
+
export declare function savePullState(s: PullState): void;
|
|
38
|
+
/** Order-independent signature of the source hashes feeding one mirrored session. */
|
|
39
|
+
export declare function sourceSignature(hashes: string[]): string;
|
|
40
|
+
interface LedgerRow {
|
|
41
|
+
size: number;
|
|
42
|
+
mtimeMs: number;
|
|
43
|
+
hash: string;
|
|
44
|
+
}
|
|
45
|
+
type Ledger = Record<string, LedgerRow>;
|
|
46
|
+
export declare function loadLedger(): Ledger;
|
|
47
|
+
export declare function saveLedger(ledger: Ledger): void;
|
|
48
|
+
/** True if the file at absPath is unchanged since we last uploaded it. */
|
|
49
|
+
export declare function ledgerUnchanged(ledger: Ledger, absPath: string, size: number, mtimeMs: number): boolean;
|
|
50
|
+
export declare function ledgerRecord(ledger: Ledger, absPath: string, size: number, mtimeMs: number, hash: string): void;
|
|
51
|
+
export {};
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Manifests and the local upload ledger.
|
|
3
|
+
*
|
|
4
|
+
* Each machine publishes one manifest object (sessions/<machine>/manifest.json)
|
|
5
|
+
* listing every session it holds, keyed by session id, with content hash + size
|
|
6
|
+
* + last event timestamp. Pulling machines diff manifests to fetch only changed
|
|
7
|
+
* objects — no per-tick LIST of thousands of keys.
|
|
8
|
+
*
|
|
9
|
+
* The local upload ledger (cache, per machine) records (size, mtime, hash) of
|
|
10
|
+
* files already pushed, so a tick re-uploads only files that actually grew.
|
|
11
|
+
*/
|
|
12
|
+
import * as fs from 'fs';
|
|
13
|
+
import * as path from 'path';
|
|
14
|
+
import * as crypto from 'crypto';
|
|
15
|
+
import { getCacheDir } from '../../state.js';
|
|
16
|
+
export function emptyManifest(machine, updatedAt) {
|
|
17
|
+
return { machine, updatedAt, agents: {} };
|
|
18
|
+
}
|
|
19
|
+
export function parseManifest(text) {
|
|
20
|
+
try {
|
|
21
|
+
const m = JSON.parse(text);
|
|
22
|
+
if (m && typeof m.machine === 'string' && m.agents && typeof m.agents === 'object')
|
|
23
|
+
return m;
|
|
24
|
+
return null;
|
|
25
|
+
}
|
|
26
|
+
catch {
|
|
27
|
+
return null;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
export function hashContent(content) {
|
|
31
|
+
return crypto.createHash('sha256').update(content).digest('hex');
|
|
32
|
+
}
|
|
33
|
+
function stateDir() {
|
|
34
|
+
return path.join(getCacheDir(), 'state', 'sessions-sync');
|
|
35
|
+
}
|
|
36
|
+
// --- local cached copy of our published manifest ---------------------------
|
|
37
|
+
function localManifestPath() {
|
|
38
|
+
return path.join(stateDir(), 'manifest.json');
|
|
39
|
+
}
|
|
40
|
+
export function loadLocalManifest() {
|
|
41
|
+
try {
|
|
42
|
+
return parseManifest(fs.readFileSync(localManifestPath(), 'utf-8'));
|
|
43
|
+
}
|
|
44
|
+
catch {
|
|
45
|
+
return null;
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
export function saveLocalManifest(m) {
|
|
49
|
+
const p = localManifestPath();
|
|
50
|
+
fs.mkdirSync(path.dirname(p), { recursive: true });
|
|
51
|
+
fs.writeFileSync(p, JSON.stringify(m), 'utf-8');
|
|
52
|
+
}
|
|
53
|
+
function pullStatePath() {
|
|
54
|
+
return path.join(stateDir(), 'pull-state.json');
|
|
55
|
+
}
|
|
56
|
+
export function loadPullState() {
|
|
57
|
+
try {
|
|
58
|
+
return JSON.parse(fs.readFileSync(pullStatePath(), 'utf-8'));
|
|
59
|
+
}
|
|
60
|
+
catch {
|
|
61
|
+
return {};
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
export function savePullState(s) {
|
|
65
|
+
const p = pullStatePath();
|
|
66
|
+
fs.mkdirSync(path.dirname(p), { recursive: true });
|
|
67
|
+
fs.writeFileSync(p, JSON.stringify(s), 'utf-8');
|
|
68
|
+
}
|
|
69
|
+
/** Order-independent signature of the source hashes feeding one mirrored session. */
|
|
70
|
+
export function sourceSignature(hashes) {
|
|
71
|
+
return [...hashes].sort().join(',');
|
|
72
|
+
}
|
|
73
|
+
function ledgerPath() {
|
|
74
|
+
return path.join(getCacheDir(), 'state', 'sessions-sync', 'upload-ledger.json');
|
|
75
|
+
}
|
|
76
|
+
export function loadLedger() {
|
|
77
|
+
try {
|
|
78
|
+
return JSON.parse(fs.readFileSync(ledgerPath(), 'utf-8'));
|
|
79
|
+
}
|
|
80
|
+
catch {
|
|
81
|
+
return {};
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
export function saveLedger(ledger) {
|
|
85
|
+
const p = ledgerPath();
|
|
86
|
+
fs.mkdirSync(path.dirname(p), { recursive: true });
|
|
87
|
+
fs.writeFileSync(p, JSON.stringify(ledger), 'utf-8');
|
|
88
|
+
}
|
|
89
|
+
/** True if the file at absPath is unchanged since we last uploaded it. */
|
|
90
|
+
export function ledgerUnchanged(ledger, absPath, size, mtimeMs) {
|
|
91
|
+
const row = ledger[absPath];
|
|
92
|
+
return !!row && row.size === size && row.mtimeMs === Math.floor(mtimeMs);
|
|
93
|
+
}
|
|
94
|
+
export function ledgerRecord(ledger, absPath, size, mtimeMs, hash) {
|
|
95
|
+
ledger[absPath] = { size, mtimeMs: Math.floor(mtimeMs), hash };
|
|
96
|
+
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Minimal S3-compatible client for Cloudflare R2, built on aws4fetch (SigV4
|
|
3
|
+
* over the platform `fetch` + WebCrypto — works identically under Bun and
|
|
4
|
+
* Node >= 22). Only the verbs sync needs: put / get / head / list / delete.
|
|
5
|
+
*
|
|
6
|
+
* No mounting, no FUSE: this is a plain object-store client driven by a polling
|
|
7
|
+
* loop, which is the only approach that is seamless on both macOS and Linux
|
|
8
|
+
* (Mountpoint for S3 is Linux-only; rclone-mount needs macFUSE).
|
|
9
|
+
*/
|
|
10
|
+
import type { R2Config } from './config.js';
|
|
11
|
+
export interface HeadResult {
|
|
12
|
+
size: number;
|
|
13
|
+
etag: string;
|
|
14
|
+
}
|
|
15
|
+
export declare class R2Client {
|
|
16
|
+
private aws;
|
|
17
|
+
private base;
|
|
18
|
+
constructor(cfg: R2Config);
|
|
19
|
+
private url;
|
|
20
|
+
/** Upload an object. Overwrites unconditionally. */
|
|
21
|
+
put(key: string, body: string | Uint8Array, contentType?: string): Promise<void>;
|
|
22
|
+
/** Fetch an object as text, or null if it does not exist (404). */
|
|
23
|
+
get(key: string): Promise<string | null>;
|
|
24
|
+
/** HEAD an object for size + etag, or null if it does not exist. */
|
|
25
|
+
head(key: string): Promise<HeadResult | null>;
|
|
26
|
+
/** Delete an object (no error if it is already absent). */
|
|
27
|
+
delete(key: string): Promise<void>;
|
|
28
|
+
/** List immediate sub-prefixes under a prefix (delimiter '/'), e.g. machine dirs. */
|
|
29
|
+
listPrefixes(prefix: string): Promise<string[]>;
|
|
30
|
+
/** List all object keys under a prefix (handles pagination). */
|
|
31
|
+
list(prefix: string): Promise<string[]>;
|
|
32
|
+
}
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Minimal S3-compatible client for Cloudflare R2, built on aws4fetch (SigV4
|
|
3
|
+
* over the platform `fetch` + WebCrypto — works identically under Bun and
|
|
4
|
+
* Node >= 22). Only the verbs sync needs: put / get / head / list / delete.
|
|
5
|
+
*
|
|
6
|
+
* No mounting, no FUSE: this is a plain object-store client driven by a polling
|
|
7
|
+
* loop, which is the only approach that is seamless on both macOS and Linux
|
|
8
|
+
* (Mountpoint for S3 is Linux-only; rclone-mount needs macFUSE).
|
|
9
|
+
*/
|
|
10
|
+
import { AwsClient } from 'aws4fetch';
|
|
11
|
+
export class R2Client {
|
|
12
|
+
aws;
|
|
13
|
+
base;
|
|
14
|
+
constructor(cfg) {
|
|
15
|
+
this.aws = new AwsClient({
|
|
16
|
+
accessKeyId: cfg.accessKeyId,
|
|
17
|
+
secretAccessKey: cfg.secretAccessKey,
|
|
18
|
+
service: 's3',
|
|
19
|
+
region: 'auto',
|
|
20
|
+
});
|
|
21
|
+
this.base = `${cfg.endpoint}/${encodeURIComponent(cfg.bucket)}`;
|
|
22
|
+
}
|
|
23
|
+
url(key) {
|
|
24
|
+
const encoded = key.split('/').map(encodeURIComponent).join('/');
|
|
25
|
+
return `${this.base}/${encoded}`;
|
|
26
|
+
}
|
|
27
|
+
/** Upload an object. Overwrites unconditionally. */
|
|
28
|
+
async put(key, body, contentType = 'application/octet-stream') {
|
|
29
|
+
const res = await this.aws.fetch(this.url(key), {
|
|
30
|
+
method: 'PUT',
|
|
31
|
+
body,
|
|
32
|
+
headers: { 'content-type': contentType },
|
|
33
|
+
});
|
|
34
|
+
if (!res.ok)
|
|
35
|
+
throw new Error(`R2 PUT ${key} failed: ${res.status} ${await safeText(res)}`);
|
|
36
|
+
}
|
|
37
|
+
/** Fetch an object as text, or null if it does not exist (404). */
|
|
38
|
+
async get(key) {
|
|
39
|
+
const res = await this.aws.fetch(this.url(key), { method: 'GET' });
|
|
40
|
+
if (res.status === 404)
|
|
41
|
+
return null;
|
|
42
|
+
if (!res.ok)
|
|
43
|
+
throw new Error(`R2 GET ${key} failed: ${res.status} ${await safeText(res)}`);
|
|
44
|
+
return await res.text();
|
|
45
|
+
}
|
|
46
|
+
/** HEAD an object for size + etag, or null if it does not exist. */
|
|
47
|
+
async head(key) {
|
|
48
|
+
const res = await this.aws.fetch(this.url(key), { method: 'HEAD' });
|
|
49
|
+
if (res.status === 404)
|
|
50
|
+
return null;
|
|
51
|
+
if (!res.ok)
|
|
52
|
+
throw new Error(`R2 HEAD ${key} failed: ${res.status}`);
|
|
53
|
+
return {
|
|
54
|
+
size: Number(res.headers.get('content-length') ?? '0'),
|
|
55
|
+
etag: (res.headers.get('etag') ?? '').replace(/"/g, ''),
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
/** Delete an object (no error if it is already absent). */
|
|
59
|
+
async delete(key) {
|
|
60
|
+
const res = await this.aws.fetch(this.url(key), { method: 'DELETE' });
|
|
61
|
+
if (!res.ok && res.status !== 404) {
|
|
62
|
+
throw new Error(`R2 DELETE ${key} failed: ${res.status}`);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
/** List immediate sub-prefixes under a prefix (delimiter '/'), e.g. machine dirs. */
|
|
66
|
+
async listPrefixes(prefix) {
|
|
67
|
+
const prefixes = [];
|
|
68
|
+
let token;
|
|
69
|
+
do {
|
|
70
|
+
const params = new URLSearchParams({ 'list-type': '2', prefix, delimiter: '/' });
|
|
71
|
+
if (token)
|
|
72
|
+
params.set('continuation-token', token);
|
|
73
|
+
const res = await this.aws.fetch(`${this.base}?${params.toString()}`, { method: 'GET' });
|
|
74
|
+
if (!res.ok)
|
|
75
|
+
throw new Error(`R2 LIST(prefixes) ${prefix} failed: ${res.status} ${await safeText(res)}`);
|
|
76
|
+
const xml = await res.text();
|
|
77
|
+
for (const m of xml.matchAll(/<CommonPrefixes><Prefix>([^<]+)<\/Prefix><\/CommonPrefixes>/g)) {
|
|
78
|
+
prefixes.push(decodeXml(m[1]));
|
|
79
|
+
}
|
|
80
|
+
const truncated = /<IsTruncated>true<\/IsTruncated>/.test(xml);
|
|
81
|
+
token = truncated ? xml.match(/<NextContinuationToken>([^<]+)<\/NextContinuationToken>/)?.[1] : undefined;
|
|
82
|
+
} while (token);
|
|
83
|
+
return prefixes;
|
|
84
|
+
}
|
|
85
|
+
/** List all object keys under a prefix (handles pagination). */
|
|
86
|
+
async list(prefix) {
|
|
87
|
+
const keys = [];
|
|
88
|
+
let token;
|
|
89
|
+
do {
|
|
90
|
+
const params = new URLSearchParams({ 'list-type': '2', prefix });
|
|
91
|
+
if (token)
|
|
92
|
+
params.set('continuation-token', token);
|
|
93
|
+
const res = await this.aws.fetch(`${this.base}?${params.toString()}`, { method: 'GET' });
|
|
94
|
+
if (!res.ok)
|
|
95
|
+
throw new Error(`R2 LIST ${prefix} failed: ${res.status} ${await safeText(res)}`);
|
|
96
|
+
const xml = await res.text();
|
|
97
|
+
for (const m of xml.matchAll(/<Key>([^<]+)<\/Key>/g)) {
|
|
98
|
+
keys.push(decodeXml(m[1]));
|
|
99
|
+
}
|
|
100
|
+
const truncated = /<IsTruncated>true<\/IsTruncated>/.test(xml);
|
|
101
|
+
token = truncated ? xml.match(/<NextContinuationToken>([^<]+)<\/NextContinuationToken>/)?.[1] : undefined;
|
|
102
|
+
} while (token);
|
|
103
|
+
return keys;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
function decodeXml(s) {
|
|
107
|
+
return s
|
|
108
|
+
.replace(/&/g, '&')
|
|
109
|
+
.replace(/</g, '<')
|
|
110
|
+
.replace(/>/g, '>')
|
|
111
|
+
.replace(/"/g, '"')
|
|
112
|
+
.replace(/'/g, "'");
|
|
113
|
+
}
|
|
114
|
+
async function safeText(res) {
|
|
115
|
+
try {
|
|
116
|
+
return (await res.text()).slice(0, 200);
|
|
117
|
+
}
|
|
118
|
+
catch {
|
|
119
|
+
return '';
|
|
120
|
+
}
|
|
121
|
+
}
|