gipity 1.0.409 → 1.0.411
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/claude-setup.js +3 -2
- package/dist/commands/add.js +10 -1
- package/dist/commands/bug.js +69 -0
- package/dist/commands/credits.js +247 -15
- package/dist/commands/job.js +3 -3
- package/dist/commands/push.js +7 -3
- package/dist/commands/relay.js +26 -2
- package/dist/commands/remove.js +9 -3
- package/dist/commands/storage.js +64 -0
- package/dist/commands/uninstall.js +2 -2
- package/dist/commands/upload.js +1 -2
- package/dist/hooks/capture-runner.js +15 -2
- package/dist/index.js +5 -4
- package/dist/knowledge.js +22 -0
- package/dist/platform.js +9 -5
- package/dist/relay/daemon.js +517 -69
- package/dist/relay/ingest-queue.js +148 -0
- package/dist/relay/installers.js +10 -2
- package/dist/relay/onboarding.js +5 -2
- package/dist/relay/redact.js +27 -4
- package/dist/relay/setup.js +4 -4
- package/dist/relay/state.js +6 -3
- package/dist/relay/stream-delta.js +235 -0
- package/dist/relay/stream-json.js +61 -15
- package/dist/setup.js +16 -7
- package/dist/sync.js +227 -106
- package/dist/updater/shim.js +4 -4
- package/dist/upload.js +1 -1
- package/package.json +6 -3
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
const DEFAULTS = {
|
|
2
|
+
maxEntries: 2000,
|
|
3
|
+
maxBytes: 8 * 1024 * 1024,
|
|
4
|
+
batchMax: 200,
|
|
5
|
+
backoffMs: [2000, 5000, 15000, 60000],
|
|
6
|
+
};
|
|
7
|
+
function entryBytes(e) {
|
|
8
|
+
try {
|
|
9
|
+
return JSON.stringify(e).length;
|
|
10
|
+
}
|
|
11
|
+
catch {
|
|
12
|
+
return 1024;
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
export class IngestQueue {
|
|
16
|
+
post;
|
|
17
|
+
queue = [];
|
|
18
|
+
bytes = 0;
|
|
19
|
+
draining = false;
|
|
20
|
+
closed = false;
|
|
21
|
+
closeDeadline = 0;
|
|
22
|
+
drainDone = Promise.resolve();
|
|
23
|
+
droppedTotal = 0;
|
|
24
|
+
markedDrop = false;
|
|
25
|
+
opts;
|
|
26
|
+
constructor(post, opts = {}) {
|
|
27
|
+
this.post = post;
|
|
28
|
+
this.opts = { ...DEFAULTS, ...opts };
|
|
29
|
+
}
|
|
30
|
+
get dropped() {
|
|
31
|
+
return this.droppedTotal;
|
|
32
|
+
}
|
|
33
|
+
/** Enqueue entries in order and kick the drainer. Sync and non-blocking. */
|
|
34
|
+
push(...entries) {
|
|
35
|
+
if (this.closed) {
|
|
36
|
+
this.opts.onWarn?.('ingest queue push after close - dropped', { count: entries.length });
|
|
37
|
+
this.droppedTotal += entries.length;
|
|
38
|
+
return;
|
|
39
|
+
}
|
|
40
|
+
for (const entry of entries) {
|
|
41
|
+
const bytes = entryBytes(entry);
|
|
42
|
+
this.queue.push({ entry, bytes });
|
|
43
|
+
this.bytes += bytes;
|
|
44
|
+
}
|
|
45
|
+
this.enforceBounds();
|
|
46
|
+
this.kick();
|
|
47
|
+
}
|
|
48
|
+
/** Stop accepting entries and drain what's left, giving up at the
|
|
49
|
+
* deadline. Returns whether everything flushed. */
|
|
50
|
+
async close(deadlineMs = 30_000) {
|
|
51
|
+
this.closed = true;
|
|
52
|
+
this.closeDeadline = Date.now() + deadlineMs;
|
|
53
|
+
this.kick();
|
|
54
|
+
// Await until the drainer has fully settled. The re-kick in `kick`'s
|
|
55
|
+
// finally is gated on `!closed`, so once we're here no NEW drain can
|
|
56
|
+
// start - a single settled await is sufficient, but loop defensively
|
|
57
|
+
// in case a kick was mid-settle when `closed` flipped.
|
|
58
|
+
while (this.draining)
|
|
59
|
+
await this.drainDone;
|
|
60
|
+
await this.drainDone;
|
|
61
|
+
const flushed = this.queue.length === 0;
|
|
62
|
+
if (!flushed) {
|
|
63
|
+
this.droppedTotal += this.queue.length;
|
|
64
|
+
this.opts.onWarn?.('ingest queue close deadline hit - entries lost', { remaining: this.queue.length });
|
|
65
|
+
this.queue = [];
|
|
66
|
+
this.bytes = 0;
|
|
67
|
+
}
|
|
68
|
+
return { flushed, dropped: this.droppedTotal };
|
|
69
|
+
}
|
|
70
|
+
enforceBounds() {
|
|
71
|
+
let droppedNow = 0;
|
|
72
|
+
while (this.queue.length > this.opts.maxEntries || this.bytes > this.opts.maxBytes) {
|
|
73
|
+
const victim = this.queue.shift();
|
|
74
|
+
if (!victim)
|
|
75
|
+
break;
|
|
76
|
+
this.bytes -= victim.bytes;
|
|
77
|
+
droppedNow++;
|
|
78
|
+
}
|
|
79
|
+
if (droppedNow > 0) {
|
|
80
|
+
this.droppedTotal += droppedNow;
|
|
81
|
+
this.opts.onWarn?.('ingest queue overflow - oldest entries dropped', { droppedNow, droppedTotal: this.droppedTotal });
|
|
82
|
+
if (!this.markedDrop) {
|
|
83
|
+
this.markedDrop = true;
|
|
84
|
+
// Visible gap marker, placed at the head so it lands before the
|
|
85
|
+
// surviving (newer) entries.
|
|
86
|
+
const marker = { kind: 'system', content: 'Some session output could not be uploaded (upload backlog overflowed).' };
|
|
87
|
+
this.queue.unshift({ entry: marker, bytes: entryBytes(marker) });
|
|
88
|
+
this.bytes += entryBytes(marker);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
kick() {
|
|
93
|
+
if (this.draining)
|
|
94
|
+
return;
|
|
95
|
+
this.draining = true;
|
|
96
|
+
this.drainDone = this.drain().finally(() => {
|
|
97
|
+
this.draining = false;
|
|
98
|
+
// Entries may have arrived while the final await of drain() settled.
|
|
99
|
+
// Never re-kick once closed: close() owns the last drain, and a
|
|
100
|
+
// re-kick here would POST a batch AFTER close() returned (the
|
|
101
|
+
// zombie-drain bug - post-after-ack + corrupted byte accounting).
|
|
102
|
+
if (this.queue.length && !this.closed)
|
|
103
|
+
this.kick();
|
|
104
|
+
});
|
|
105
|
+
}
|
|
106
|
+
async drain() {
|
|
107
|
+
const sleep = this.opts.sleep ?? ((ms) => new Promise(r => setTimeout(r, ms)));
|
|
108
|
+
let attempt = 0;
|
|
109
|
+
while (this.queue.length) {
|
|
110
|
+
// Remove the batch from the queue BEFORE awaiting the POST. A push
|
|
111
|
+
// that overflows the bounds during the await runs `enforceBounds`,
|
|
112
|
+
// which shifts the queue head - if the in-flight batch were still
|
|
113
|
+
// at the head, a later splice-by-index would remove the wrong
|
|
114
|
+
// entries and drift `bytes`. Holding the batch out of the queue
|
|
115
|
+
// makes overflow touch only not-in-flight entries.
|
|
116
|
+
const batch = this.queue.splice(0, this.opts.batchMax);
|
|
117
|
+
const batchBytes = batch.reduce((n, b) => n + b.bytes, 0);
|
|
118
|
+
this.bytes -= batchBytes;
|
|
119
|
+
const result = await this.post(batch.map(b => b.entry));
|
|
120
|
+
if (result.ok) {
|
|
121
|
+
attempt = 0;
|
|
122
|
+
continue;
|
|
123
|
+
}
|
|
124
|
+
// Definitive rejection (4xx: schema or auth) - a replay can never
|
|
125
|
+
// succeed, and retrying would stall the queue behind a poisoned
|
|
126
|
+
// batch. Drop it (counted + warned) and move on.
|
|
127
|
+
if (result.retryable === false) {
|
|
128
|
+
attempt = 0;
|
|
129
|
+
this.droppedTotal += batch.length;
|
|
130
|
+
this.opts.onWarn?.('ingest batch rejected by server - dropped', { count: batch.length });
|
|
131
|
+
continue;
|
|
132
|
+
}
|
|
133
|
+
// Transient failure: put the batch back at the head and retry after
|
|
134
|
+
// backoff (order preserved; source_uuid dedup makes a partial
|
|
135
|
+
// server-side success safe).
|
|
136
|
+
this.queue.unshift(...batch);
|
|
137
|
+
this.bytes += batchBytes;
|
|
138
|
+
const backoff = this.opts.backoffMs[Math.min(attempt, this.opts.backoffMs.length - 1)];
|
|
139
|
+
attempt++;
|
|
140
|
+
if (this.closed && Date.now() + backoff > this.closeDeadline)
|
|
141
|
+
return;
|
|
142
|
+
await sleep(backoff);
|
|
143
|
+
if (this.closed && Date.now() > this.closeDeadline)
|
|
144
|
+
return;
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
//# sourceMappingURL=ingest-queue.js.map
|
package/dist/relay/installers.js
CHANGED
|
@@ -60,7 +60,15 @@ function launchdPlan(cliPath) {
|
|
|
60
60
|
<string>run</string>
|
|
61
61
|
</array>
|
|
62
62
|
<key>RunAtLoad</key><true/>
|
|
63
|
-
|
|
63
|
+
<!-- Restart ONLY on a non-zero exit, mirroring systemd's Restart=on-failure.
|
|
64
|
+
The daemon exits 0 on a clean revoke/shutdown; a bare KeepAlive=true
|
|
65
|
+
would relaunch it anyway, and run() would then silently re-register a
|
|
66
|
+
new device - undoing the revoke. SuccessfulExit=false keeps crash
|
|
67
|
+
recovery without that re-pair loop. -->
|
|
68
|
+
<key>KeepAlive</key>
|
|
69
|
+
<dict>
|
|
70
|
+
<key>SuccessfulExit</key><false/>
|
|
71
|
+
</dict>
|
|
64
72
|
<key>StandardOutPath</key><string>${join(logDir, 'relay.out.log')}</string>
|
|
65
73
|
<key>StandardErrorPath</key><string>${join(logDir, 'relay.err.log')}</string>
|
|
66
74
|
<key>ProcessType</key><string>Background</string>
|
|
@@ -80,7 +88,7 @@ function launchdPlan(cliPath) {
|
|
|
80
88
|
enableDisplay: display(enableCmds),
|
|
81
89
|
disableDisplay: display(disableCmds),
|
|
82
90
|
statusDisplay: statusCmd.join(' '),
|
|
83
|
-
summary: 'LaunchAgent at ~/Library/LaunchAgents/ai.gipity.relay.plist (starts at login,
|
|
91
|
+
summary: 'LaunchAgent at ~/Library/LaunchAgents/ai.gipity.relay.plist (starts at login, restarts on failure)',
|
|
84
92
|
};
|
|
85
93
|
}
|
|
86
94
|
// ─── Linux - systemd --user unit ───────────────────────────────────────
|
package/dist/relay/onboarding.js
CHANGED
|
@@ -56,8 +56,11 @@ export async function maybeOfferRelayOn() {
|
|
|
56
56
|
}
|
|
57
57
|
catch (err) {
|
|
58
58
|
console.error(`\n ${clrError(`Could not create device: ${err?.message || err}`)}`);
|
|
59
|
-
console.error(` ${dim('Skipping
|
|
60
|
-
|
|
59
|
+
console.error(` ${dim('Skipping for now - we\'ll offer again next time. Or turn it on with `gipity relay install`.')}`);
|
|
60
|
+
// Deliberately DON'T persist relay_enabled here: the user SAID YES, and
|
|
61
|
+
// this is a transient failure (network blip, server down). Leaving the
|
|
62
|
+
// tri-state unset means onboarding re-offers on the next `gipity claude`
|
|
63
|
+
// run instead of silently opting the user out forever.
|
|
61
64
|
return;
|
|
62
65
|
}
|
|
63
66
|
// Start the daemon for this session.
|
package/dist/relay/redact.js
CHANGED
|
@@ -38,15 +38,38 @@ const JWT_RE = /eyJ[A-Za-z0-9_-]+\.eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+/g;
|
|
|
38
38
|
* literal-secret list. An `sk-ant-` token in a relay transcript is always a
|
|
39
39
|
* credential, so over-redaction risk is negligible. */
|
|
40
40
|
const ANTHROPIC_KEY_RE = /sk-ant-[A-Za-z0-9_-]{20,}/g;
|
|
41
|
-
/**
|
|
42
|
-
*
|
|
43
|
-
|
|
41
|
+
/** Well-known third-party credential shapes. A `bypassPermissions` session can
|
|
42
|
+
* read the host's environment and files, so a `cat .env` / `env` could echo
|
|
43
|
+
* ANY of these into the transcript - not just the Gipity/Anthropic tokens on
|
|
44
|
+
* the literal-secret list. Each pattern is a high-entropy, provider-specific
|
|
45
|
+
* prefix that is effectively never a false positive in a relay transcript, so
|
|
46
|
+
* the over-redaction risk is the same negligible trade-off already accepted
|
|
47
|
+
* for the JWT and `sk-ant-` passes. Extend this list as new providers appear;
|
|
48
|
+
* it is defense-in-depth, NOT a guarantee (a base64/chunked secret still
|
|
49
|
+
* slips through - the real backstop remains an instantly-revocable
|
|
50
|
+
* credential). */
|
|
51
|
+
const THIRD_PARTY_KEY_RES = [
|
|
52
|
+
/sk-[A-Za-z0-9_-]{20,}/g, // OpenAI (sk-, sk-proj-) + any sk- key
|
|
53
|
+
/\b(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{36,}\b/g, // GitHub tokens
|
|
54
|
+
/\bgithub_pat_[A-Za-z0-9_]{60,}\b/g, // GitHub fine-grained PAT
|
|
55
|
+
/\bAKIA[0-9A-Z]{16}\b/g, // AWS access key id
|
|
56
|
+
/\b(?:sk|rk)_live_[A-Za-z0-9]{20,}\b/g, // Stripe live secret/restricted key
|
|
57
|
+
/\bxox[baprs]-[A-Za-z0-9-]{10,}\b/g, // Slack tokens
|
|
58
|
+
/\bAIza[A-Za-z0-9_-]{35}\b/g, // Google API key
|
|
59
|
+
/-----BEGIN (?:RSA |EC |OPENSSH |DSA |PGP )?PRIVATE KEY-----[\s\S]*?-----END (?:RSA |EC |OPENSSH |DSA |PGP )?PRIVATE KEY-----/g, // PEM private keys
|
|
60
|
+
];
|
|
61
|
+
/** Replace every occurrence of each known secret - plus any JWT-, Anthropic-,
|
|
62
|
+
* or well-known-third-party-key-shaped substring - in `text` with the marker. */
|
|
63
|
+
export function redactString(text, secrets) {
|
|
44
64
|
let out = text;
|
|
45
65
|
for (const secret of secrets) {
|
|
46
66
|
if (out.includes(secret))
|
|
47
67
|
out = out.split(secret).join(REDACTION_MARKER);
|
|
48
68
|
}
|
|
49
|
-
|
|
69
|
+
out = out.replace(JWT_RE, REDACTION_MARKER).replace(ANTHROPIC_KEY_RE, REDACTION_MARKER);
|
|
70
|
+
for (const re of THIRD_PARTY_KEY_RES)
|
|
71
|
+
out = out.replace(re, REDACTION_MARKER);
|
|
72
|
+
return out;
|
|
50
73
|
}
|
|
51
74
|
/** Deep-walk any JSON-ish value, redacting every string. Returns a new
|
|
52
75
|
* value; objects/arrays are cloned, primitives passed through. */
|
package/dist/relay/setup.js
CHANGED
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
* the interactive prompts that live in `onboarding.ts`. No `console` output
|
|
11
11
|
* and no `process.exit` in this module: callers own all UX.
|
|
12
12
|
*/
|
|
13
|
-
import {
|
|
13
|
+
import { spawnCommand, spawnSyncCommand } from '../platform.js';
|
|
14
14
|
import { hostname, platform as osPlatform } from 'os';
|
|
15
15
|
import { resolve, dirname } from 'path';
|
|
16
16
|
import { mkdirSync, writeFileSync } from 'fs';
|
|
@@ -87,7 +87,7 @@ export function startDaemon() {
|
|
|
87
87
|
if (state.isDaemonRunning())
|
|
88
88
|
return;
|
|
89
89
|
try {
|
|
90
|
-
const child =
|
|
90
|
+
const child = spawnCommand(process.execPath, [resolveCliPath(), 'relay', 'run'], {
|
|
91
91
|
detached: true,
|
|
92
92
|
stdio: 'ignore',
|
|
93
93
|
});
|
|
@@ -118,7 +118,7 @@ export function installAutostart(opts = {}) {
|
|
|
118
118
|
writeFileSync(plan.path, plan.content);
|
|
119
119
|
let ok = true;
|
|
120
120
|
for (const argv of plan.enableCmds) {
|
|
121
|
-
const r =
|
|
121
|
+
const r = spawnSyncCommand(argv[0], argv.slice(1), { stdio });
|
|
122
122
|
if (r.status !== 0) {
|
|
123
123
|
ok = false;
|
|
124
124
|
break;
|
|
@@ -136,7 +136,7 @@ export function removeAutostart(opts = {}) {
|
|
|
136
136
|
const plan = planFor({ cliPath: resolveCliPath() });
|
|
137
137
|
let ok = true;
|
|
138
138
|
for (const argv of plan.disableCmds) {
|
|
139
|
-
const r =
|
|
139
|
+
const r = spawnSyncCommand(argv[0], argv.slice(1), { stdio });
|
|
140
140
|
if (r.status !== 0)
|
|
141
141
|
ok = false;
|
|
142
142
|
}
|
package/dist/relay/state.js
CHANGED
|
@@ -47,9 +47,12 @@ export function loadState() {
|
|
|
47
47
|
}
|
|
48
48
|
export function saveState(state) {
|
|
49
49
|
mkdirSync(RELAY_DIR, { recursive: true });
|
|
50
|
-
|
|
51
|
-
//
|
|
52
|
-
//
|
|
50
|
+
// `mode` on write means a NEWLY-created file is owner-only from the first
|
|
51
|
+
// byte - no window where it exists at the umask default (typically 0644)
|
|
52
|
+
// before a follow-up chmod tightens it. The chmodSync still runs to fix a
|
|
53
|
+
// file that already existed with looser permissions (mode is ignored for
|
|
54
|
+
// an existing file).
|
|
55
|
+
writeFileSync(RELAY_FILE, JSON.stringify(state, null, 2) + '\n', { mode: FILE_MODE });
|
|
53
56
|
try {
|
|
54
57
|
chmodSync(RELAY_FILE, FILE_MODE);
|
|
55
58
|
}
|
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Token-streaming delta pipeline for the relay daemon.
|
|
3
|
+
*
|
|
4
|
+
* With `--include-partial-messages`, Claude Code emits `stream_event`
|
|
5
|
+
* envelopes (content_block_start / content_block_delta / content_block_stop)
|
|
6
|
+
* carrying text, thinking, and tool-input fragments as they generate. The
|
|
7
|
+
* daemon forwards them - throttled and redacted - to
|
|
8
|
+
* `POST /remote-sessions/:convGuid/stream-delta`, which the server
|
|
9
|
+
* broadcasts to the web CLI as `dispatch:delta`. Purely ephemeral: no DB
|
|
10
|
+
* write, no retry; the persisted assistant/tool entries (whole events)
|
|
11
|
+
* remain the source of truth and replace the streamed view on the next
|
|
12
|
+
* refresh.
|
|
13
|
+
*
|
|
14
|
+
* Redaction across chunk boundaries: a secret split over two fragments
|
|
15
|
+
* would defeat a per-fragment scrub. The accumulator therefore only ever
|
|
16
|
+
* emits up to the last WHITESPACE boundary (holding back the trailing
|
|
17
|
+
* partial token until more input or block close) and scrubs each emitted
|
|
18
|
+
* span. Secrets in our threat set (host OAuth/API/device tokens, JWTs,
|
|
19
|
+
* provider keys) are contiguous non-whitespace tokens, so a held-back
|
|
20
|
+
* partial token can never leak a complete secret and a completed token is
|
|
21
|
+
* always scrubbed whole. Multi-line PEM blocks are the one shape this
|
|
22
|
+
* can't fully hold back - accepted on this ephemeral channel (the
|
|
23
|
+
* persisted path fully redacts, and the streamed text is replaced by the
|
|
24
|
+
* redacted stored message within seconds).
|
|
25
|
+
*/
|
|
26
|
+
import { redactString } from './redact.js';
|
|
27
|
+
/** A trailing token longer than this is emitted anyway (minified code,
|
|
28
|
+
* base64 blobs) - holding back indefinitely would stall the stream. No
|
|
29
|
+
* credential in our threat set is this long. */
|
|
30
|
+
const MAX_HOLDBACK = 4096;
|
|
31
|
+
/** PEM key markers. A private key block spans multiple whitespace-
|
|
32
|
+
* separated lines, so the whitespace-boundary holdback would emit
|
|
33
|
+
* `BEGIN` before `END` arrives and per-span redaction would miss it.
|
|
34
|
+
* emitUpTo holds the whole block back until END so it scrubs whole. */
|
|
35
|
+
const PEM_BEGIN = /-----BEGIN [A-Z ]*PRIVATE KEY-----/;
|
|
36
|
+
const PEM_END = /-----END [A-Z ]*PRIVATE KEY-----/;
|
|
37
|
+
/** Emit boundary: everything up to (and including) the last whitespace.
|
|
38
|
+
* `closing` (block stop) emits everything. */
|
|
39
|
+
export function safeEmitBoundary(s, closing) {
|
|
40
|
+
if (closing)
|
|
41
|
+
return s.length;
|
|
42
|
+
const m = s.match(/\S+$/);
|
|
43
|
+
if (!m)
|
|
44
|
+
return s.length;
|
|
45
|
+
if (m[0].length > MAX_HOLDBACK)
|
|
46
|
+
return s.length;
|
|
47
|
+
return s.length - m[0].length;
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Per-spawn accumulator: feed it raw stream_event envelopes, it returns
|
|
51
|
+
* redaction-safe StreamDeltaEvents ready to batch. Skips subagent events
|
|
52
|
+
* (parent_tool_use_id set) - those belong to the nested Task timeline,
|
|
53
|
+
* not the main stream zone.
|
|
54
|
+
*/
|
|
55
|
+
export class DeltaAccumulator {
|
|
56
|
+
getSecrets;
|
|
57
|
+
blocks = new Map();
|
|
58
|
+
/** The in-flight message's id, captured from `message_start` -
|
|
59
|
+
* content_block_* events don't carry it themselves. This is what makes
|
|
60
|
+
* the client-side finalize swap line up: persisted assistant rows are
|
|
61
|
+
* keyed `msg_id#n` (Phase 2 source_uuid), streamed blocks `msg_id:n`. */
|
|
62
|
+
currentMessageId = 'msg';
|
|
63
|
+
constructor(getSecrets) {
|
|
64
|
+
this.getSecrets = getSecrets;
|
|
65
|
+
}
|
|
66
|
+
/** Map one parsed stream-json event to zero or more delta events. */
|
|
67
|
+
note(evt) {
|
|
68
|
+
if (evt.type !== 'stream_event')
|
|
69
|
+
return [];
|
|
70
|
+
if (evt.parent_tool_use_id)
|
|
71
|
+
return []; // subagent stream - not ours (Phase 6)
|
|
72
|
+
const inner = evt.event;
|
|
73
|
+
if (!inner || typeof inner.type !== 'string')
|
|
74
|
+
return [];
|
|
75
|
+
if (inner.type === 'message_start') {
|
|
76
|
+
if (typeof inner.message?.id === 'string' && inner.message.id) {
|
|
77
|
+
this.currentMessageId = inner.message.id;
|
|
78
|
+
}
|
|
79
|
+
return [];
|
|
80
|
+
}
|
|
81
|
+
const messageId = this.currentMessageId;
|
|
82
|
+
if (inner.type === 'content_block_start') {
|
|
83
|
+
const idx = typeof inner.index === 'number' ? inner.index : 0;
|
|
84
|
+
const cb = inner.content_block ?? {};
|
|
85
|
+
const type = cb.type === 'text' ? 'text' : cb.type === 'thinking' ? 'thinking' : cb.type === 'tool_use' ? 'tool_use' : null;
|
|
86
|
+
if (!type)
|
|
87
|
+
return [];
|
|
88
|
+
this.blocks.set(`${messageId}:${idx}`, { type, raw: '', rawEmitted: 0, outEmitted: 0, started: true });
|
|
89
|
+
const out = { message_id: messageId, block_index: idx, block_type: type, kind: 'start' };
|
|
90
|
+
if (type === 'tool_use' && typeof cb.name === 'string')
|
|
91
|
+
out.tool_name = cb.name;
|
|
92
|
+
return [out];
|
|
93
|
+
}
|
|
94
|
+
if (inner.type === 'content_block_delta') {
|
|
95
|
+
const idx = typeof inner.index === 'number' ? inner.index : 0;
|
|
96
|
+
const d = inner.delta ?? {};
|
|
97
|
+
const fragment = typeof d.text === 'string' ? d.text
|
|
98
|
+
: typeof d.thinking === 'string' ? d.thinking
|
|
99
|
+
: typeof d.partial_json === 'string' ? d.partial_json
|
|
100
|
+
: null;
|
|
101
|
+
if (fragment === null || fragment === '')
|
|
102
|
+
return [];
|
|
103
|
+
const key = `${messageId}:${idx}`;
|
|
104
|
+
let b = this.blocks.get(key);
|
|
105
|
+
if (!b) {
|
|
106
|
+
// Delta without a start (daemon restarted mid-message or an
|
|
107
|
+
// unknown block type at start time) - infer the type.
|
|
108
|
+
const type = typeof d.partial_json === 'string' ? 'tool_use'
|
|
109
|
+
: typeof d.thinking === 'string' ? 'thinking' : 'text';
|
|
110
|
+
b = { type, raw: '', rawEmitted: 0, outEmitted: 0, started: false };
|
|
111
|
+
this.blocks.set(key, b);
|
|
112
|
+
}
|
|
113
|
+
b.raw += fragment;
|
|
114
|
+
return this.emitUpTo(messageId, idx, b, false);
|
|
115
|
+
}
|
|
116
|
+
if (inner.type === 'content_block_stop') {
|
|
117
|
+
const idx = typeof inner.index === 'number' ? inner.index : 0;
|
|
118
|
+
const key = `${messageId}:${idx}`;
|
|
119
|
+
const b = this.blocks.get(key);
|
|
120
|
+
if (!b)
|
|
121
|
+
return [];
|
|
122
|
+
const out = this.emitUpTo(messageId, idx, b, true);
|
|
123
|
+
out.push({ message_id: messageId, block_index: idx, block_type: b.type, kind: 'stop' });
|
|
124
|
+
this.blocks.delete(key);
|
|
125
|
+
return out;
|
|
126
|
+
}
|
|
127
|
+
return [];
|
|
128
|
+
}
|
|
129
|
+
emitUpTo(messageId, idx, b, closing) {
|
|
130
|
+
// Scan only the not-yet-emitted tail: the boundary can never precede
|
|
131
|
+
// rawEmitted (already a whitespace boundary), and running the regex
|
|
132
|
+
// over the full accumulated `raw` on every fragment is O(n²) over a
|
|
133
|
+
// long block - a big minified/base64 tool-input block would stall the
|
|
134
|
+
// daemon's event loop (which also drives the ingest queue + ticks).
|
|
135
|
+
const tail = b.raw.slice(b.rawEmitted);
|
|
136
|
+
// PEM keys are the one multi-LINE secret the whitespace-boundary
|
|
137
|
+
// holdback doesn't catch: `-----BEGIN … PRIVATE KEY-----` and its
|
|
138
|
+
// `-----END-----` land in different spans, so per-span redactString
|
|
139
|
+
// never sees them together. When an unterminated BEGIN marker is in
|
|
140
|
+
// the pending tail, hold the WHOLE tail back until END arrives (or the
|
|
141
|
+
// block closes), so redactString scrubs the complete block in one span.
|
|
142
|
+
if (!closing && PEM_BEGIN.test(tail) && !PEM_END.test(tail))
|
|
143
|
+
return [];
|
|
144
|
+
const tailBoundary = safeEmitBoundary(tail, closing);
|
|
145
|
+
if (tailBoundary <= 0)
|
|
146
|
+
return [];
|
|
147
|
+
const span = tail.slice(0, tailBoundary);
|
|
148
|
+
const boundary = b.rawEmitted + tailBoundary;
|
|
149
|
+
b.rawEmitted = boundary;
|
|
150
|
+
const scrubbed = redactString(span, this.getSecrets());
|
|
151
|
+
if (!scrubbed)
|
|
152
|
+
return [];
|
|
153
|
+
const evt = {
|
|
154
|
+
message_id: messageId,
|
|
155
|
+
block_index: idx,
|
|
156
|
+
block_type: b.type,
|
|
157
|
+
kind: 'delta',
|
|
158
|
+
text: scrubbed,
|
|
159
|
+
acc_offset: b.outEmitted,
|
|
160
|
+
};
|
|
161
|
+
b.outEmitted += scrubbed.length;
|
|
162
|
+
return [evt];
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
// ─── Throttled batcher ──────────────────────────────────────────────────
|
|
166
|
+
const FLUSH_INTERVAL_MS = 150;
|
|
167
|
+
const FLUSH_BYTES = 4096;
|
|
168
|
+
/** Server-side caps (mirror the /stream-delta Zod schema). */
|
|
169
|
+
const MAX_EVENTS_PER_FLUSH = 64;
|
|
170
|
+
const MAX_TEXT_PER_FLUSH = 16_384;
|
|
171
|
+
/**
|
|
172
|
+
* Buffers delta events and flushes them as numbered batches every 150ms
|
|
173
|
+
* or 4KB, whichever first. Fire-and-forget by design: a lost flush
|
|
174
|
+
* self-heals (the client detects the `acc_offset` gap and the next
|
|
175
|
+
* refresh replaces the stream zone with the stored message anyway).
|
|
176
|
+
*/
|
|
177
|
+
export class DeltaBatcher {
|
|
178
|
+
send;
|
|
179
|
+
intervalMs;
|
|
180
|
+
buffer = [];
|
|
181
|
+
bufferedText = 0;
|
|
182
|
+
seq = 0;
|
|
183
|
+
timer = null;
|
|
184
|
+
closed = false;
|
|
185
|
+
constructor(send, intervalMs = FLUSH_INTERVAL_MS) {
|
|
186
|
+
this.send = send;
|
|
187
|
+
this.intervalMs = intervalMs;
|
|
188
|
+
}
|
|
189
|
+
push(events) {
|
|
190
|
+
if (this.closed || events.length === 0)
|
|
191
|
+
return;
|
|
192
|
+
for (const e of events) {
|
|
193
|
+
this.buffer.push(e);
|
|
194
|
+
this.bufferedText += e.text?.length ?? 0;
|
|
195
|
+
}
|
|
196
|
+
if (this.bufferedText >= FLUSH_BYTES || this.buffer.length >= MAX_EVENTS_PER_FLUSH) {
|
|
197
|
+
this.flush();
|
|
198
|
+
return;
|
|
199
|
+
}
|
|
200
|
+
if (!this.timer) {
|
|
201
|
+
this.timer = setTimeout(() => this.flush(), this.intervalMs);
|
|
202
|
+
this.timer.unref?.();
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
flush() {
|
|
206
|
+
if (this.timer) {
|
|
207
|
+
clearTimeout(this.timer);
|
|
208
|
+
this.timer = null;
|
|
209
|
+
}
|
|
210
|
+
while (this.buffer.length) {
|
|
211
|
+
// Respect both server caps per POST: event count and total text.
|
|
212
|
+
const batch = [];
|
|
213
|
+
let text = 0;
|
|
214
|
+
while (this.buffer.length && batch.length < MAX_EVENTS_PER_FLUSH) {
|
|
215
|
+
const next = this.buffer[0];
|
|
216
|
+
const nextLen = next.text?.length ?? 0;
|
|
217
|
+
if (batch.length > 0 && text + nextLen > MAX_TEXT_PER_FLUSH)
|
|
218
|
+
break;
|
|
219
|
+
// A single oversized event gets truncated rather than wedging the queue.
|
|
220
|
+
if (nextLen > MAX_TEXT_PER_FLUSH)
|
|
221
|
+
next.text = next.text.slice(0, MAX_TEXT_PER_FLUSH);
|
|
222
|
+
batch.push(this.buffer.shift());
|
|
223
|
+
text += next.text?.length ?? 0;
|
|
224
|
+
}
|
|
225
|
+
this.bufferedText = Math.max(0, this.bufferedText - text);
|
|
226
|
+
this.send({ seq: this.seq++, events: batch });
|
|
227
|
+
}
|
|
228
|
+
this.bufferedText = 0;
|
|
229
|
+
}
|
|
230
|
+
close() {
|
|
231
|
+
this.flush();
|
|
232
|
+
this.closed = true;
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
//# sourceMappingURL=stream-delta.js.map
|
|
@@ -58,31 +58,61 @@ function joinAssistantText(content) {
|
|
|
58
58
|
}
|
|
59
59
|
return parts.join('\n');
|
|
60
60
|
}
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
* `tool_result` arrives).
|
|
66
|
-
*
|
|
67
|
-
* `toolNames` (optional) is a per-dispatch `tool_use_id → tool_name`
|
|
68
|
-
* map threaded across events by the daemon: assistant events record
|
|
69
|
-
* each tool call's name, the later user event with the paired
|
|
70
|
-
* `tool_result` reads it back so the server can denormalize `tool_name`
|
|
71
|
-
* onto the tool row even when the result lands as a stub. */
|
|
72
|
-
export function mapEventToEntries(evt, toolNames) {
|
|
61
|
+
export function mapEventToEntries(evt, ctx) {
|
|
62
|
+
// Back-compat: older call sites (and tests) pass the bare toolNames map.
|
|
63
|
+
const c = ctx instanceof Map ? { toolNames: ctx } : (ctx ?? {});
|
|
64
|
+
const toolNames = c.toolNames;
|
|
73
65
|
const out = [];
|
|
74
66
|
if (evt.type === 'system' && evt.subtype === 'init') {
|
|
75
67
|
const s = evt;
|
|
76
|
-
if (s.session_id)
|
|
77
|
-
out.push({
|
|
68
|
+
if (s.session_id) {
|
|
69
|
+
out.push({
|
|
70
|
+
kind: 'attach',
|
|
71
|
+
session_id: s.session_id,
|
|
72
|
+
cwd: s.cwd,
|
|
73
|
+
source: 'startup',
|
|
74
|
+
model: typeof s.model === 'string' ? s.model : undefined,
|
|
75
|
+
tools_count: Array.isArray(s.tools) ? s.tools.length : undefined,
|
|
76
|
+
mcp_count: Array.isArray(s.mcp_servers) ? s.mcp_servers.length : undefined,
|
|
77
|
+
api_key_source: typeof s.apiKeySource === 'string' ? s.apiKeySource : undefined,
|
|
78
|
+
});
|
|
79
|
+
}
|
|
78
80
|
return out;
|
|
79
81
|
}
|
|
82
|
+
// Context compaction boundary - the server has a `compact` kind for this
|
|
83
|
+
// (renders as a "context compacted" marker); nothing emitted it before.
|
|
84
|
+
if (evt.type === 'system' && evt.subtype === 'compact_boundary') {
|
|
85
|
+
const trigger = evt.compact_metadata?.trigger ?? evt.trigger;
|
|
86
|
+
out.push({ kind: 'compact', trigger: typeof trigger === 'string' ? trigger : undefined });
|
|
87
|
+
return out;
|
|
88
|
+
}
|
|
89
|
+
// Subagent attribution: the envelope carries the spawning Task/Agent
|
|
90
|
+
// tool_use id on every event from that subagent's turn.
|
|
91
|
+
const parentToolUseId = typeof evt.parent_tool_use_id === 'string'
|
|
92
|
+
? evt.parent_tool_use_id : undefined;
|
|
80
93
|
if (evt.type === 'assistant') {
|
|
81
94
|
const msg = evt.message ?? {};
|
|
82
95
|
const content = Array.isArray(msg.content) ? msg.content : [];
|
|
83
96
|
const text = joinAssistantText(content);
|
|
84
97
|
if (text || content.length) {
|
|
85
|
-
|
|
98
|
+
const entry = { kind: 'assistant', text, blocks: content, ...usageFields(msg), parent_tool_use_id: parentToolUseId };
|
|
99
|
+
// source_uuid = `${msg.id}#${eventSeq}`. The client's stream-zone
|
|
100
|
+
// finalize-swap matches this against its block key `${msg.id}:${block_index}`,
|
|
101
|
+
// i.e. it relies on eventSeq === block_index. That holds because
|
|
102
|
+
// Claude Code (with --include-partial-messages) emits exactly ONE
|
|
103
|
+
// assistant event per content block, in index order (verified
|
|
104
|
+
// against CLI v2.1.199 in the Phase 0 spike: a thinking block and a
|
|
105
|
+
// text block arrived as two separate assistant events, seq 0 and 1,
|
|
106
|
+
// matching block indices 0 and 1). If a future CLI ever packs two
|
|
107
|
+
// blocks into one assistant event, seq lags index for later blocks
|
|
108
|
+
// and those streamed blocks aren't removed until the ack wipes the
|
|
109
|
+
// zone - a cosmetic, self-healing duplication, not data loss.
|
|
110
|
+
if (typeof msg.id === 'string' && msg.id && c.msgSeq) {
|
|
111
|
+
const n = c.msgSeq.get(msg.id) ?? 0;
|
|
112
|
+
c.msgSeq.set(msg.id, n + 1);
|
|
113
|
+
entry.source_uuid = `${msg.id}#${n}`;
|
|
114
|
+
}
|
|
115
|
+
out.push(entry);
|
|
86
116
|
}
|
|
87
117
|
for (const block of content) {
|
|
88
118
|
if (block?.type === 'tool_use' && typeof block.id === 'string') {
|
|
@@ -93,6 +123,8 @@ export function mapEventToEntries(evt, toolNames) {
|
|
|
93
123
|
tool_use_id: block.id,
|
|
94
124
|
tool_name: toolName,
|
|
95
125
|
tool_input: block.input ?? null,
|
|
126
|
+
source_uuid: block.id,
|
|
127
|
+
parent_tool_use_id: parentToolUseId,
|
|
96
128
|
});
|
|
97
129
|
}
|
|
98
130
|
}
|
|
@@ -109,6 +141,7 @@ export function mapEventToEntries(evt, toolNames) {
|
|
|
109
141
|
tool_name: toolNames?.get(block.tool_use_id),
|
|
110
142
|
content: block.content ?? null,
|
|
111
143
|
is_error: Boolean(block.is_error),
|
|
144
|
+
parent_tool_use_id: parentToolUseId,
|
|
112
145
|
});
|
|
113
146
|
}
|
|
114
147
|
}
|
|
@@ -130,6 +163,19 @@ export function mapEventToEntries(evt, toolNames) {
|
|
|
130
163
|
}
|
|
131
164
|
return out;
|
|
132
165
|
}
|
|
166
|
+
// Deliberately unmapped chatter (hook lifecycle, per-chunk thinking
|
|
167
|
+
// token counts, rate-limit notices, and stream_event token deltas -
|
|
168
|
+
// the latter are handled by the separate stream-delta pipeline) -
|
|
169
|
+
// skipped without accounting so a normal session doesn't log dozens of
|
|
170
|
+
// "unmapped" entries. Everything else unknown IS accounted via
|
|
171
|
+
// onUnmapped so a new Claude Code event type shows up in the daemon
|
|
172
|
+
// log instead of vanishing silently.
|
|
173
|
+
const subtype = evt.subtype;
|
|
174
|
+
const KNOWN_NOISE = new Set(['hook_started', 'hook_response', 'thinking_tokens', 'status', 'task_started', 'task_progress', 'task_updated', 'task_notification']);
|
|
175
|
+
if (!(evt.type === 'system' && subtype && KNOWN_NOISE.has(subtype))
|
|
176
|
+
&& evt.type !== 'rate_limit_event' && evt.type !== 'stream_event') {
|
|
177
|
+
c.onUnmapped?.(evt.type, subtype);
|
|
178
|
+
}
|
|
133
179
|
return out;
|
|
134
180
|
}
|
|
135
181
|
/** Pull token usage + model + stop_reason off an assistant stream `message`.
|