@ccmsg/cli 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +23 -0
- package/package.json +32 -0
- package/src/cli.ts +1074 -0
- package/src/daemon/control.ts +88 -0
- package/src/daemon/index.ts +6 -0
- package/src/daemon/link.ts +93 -0
- package/src/daemon/log.ts +116 -0
- package/src/daemon/registry.ts +285 -0
- package/src/daemon/snapshot.ts +115 -0
- package/src/daemon/supervise.ts +446 -0
- package/src/dispatch/caller.ts +47 -0
- package/src/dispatch/dispatch.ts +128 -0
- package/src/dispatch/handler.ts +55 -0
- package/src/dispatch/identity.ts +22 -0
- package/src/dispatch/index.ts +5 -0
- package/src/dispatch/result.ts +58 -0
- package/src/files/containment.ts +263 -0
- package/src/files/files.ts +421 -0
- package/src/files/index.ts +14 -0
- package/src/files/sandbox.ts +0 -0
- package/src/greeting/hook.ts +48 -0
- package/src/greeting/index.ts +2 -0
- package/src/greeting/meta.ts +66 -0
- package/src/instance/config.ts +424 -0
- package/src/instance/handlers.ts +28 -0
- package/src/instance/identity.ts +44 -0
- package/src/instance/index.ts +8 -0
- package/src/instance/instance.ts +911 -0
- package/src/instance/lock.ts +108 -0
- package/src/instance/log.ts +30 -0
- package/src/instance/paths.ts +200 -0
- package/src/instance/socket.ts +62 -0
- package/src/kv/index.ts +2 -0
- package/src/kv/merge.ts +66 -0
- package/src/kv/store.ts +195 -0
- package/src/launcher/index.ts +4 -0
- package/src/launcher/launcher.ts +190 -0
- package/src/launcher/roots.ts +32 -0
- package/src/launcher/spawn.ts +81 -0
- package/src/launcher/tree.ts +80 -0
- package/src/mesh/index.ts +5 -0
- package/src/mesh/keys.ts +158 -0
- package/src/mesh/mesh.ts +1169 -0
- package/src/mesh/probe.ts +100 -0
- package/src/mesh/relay.ts +147 -0
- package/src/mesh/wire.ts +96 -0
- package/src/messaging/delivery.ts +375 -0
- package/src/messaging/direct.ts +433 -0
- package/src/messaging/handlers.ts +14 -0
- package/src/messaging/inbox.ts +191 -0
- package/src/messaging/index.ts +5 -0
- package/src/messaging/notify.ts +117 -0
- package/src/plugin/claude.ts +148 -0
- package/src/plugin/index.ts +13 -0
- package/src/plugin/install.ts +416 -0
- package/src/service/index.ts +1 -0
- package/src/service/service.ts +359 -0
- package/src/sessions/classify.ts +66 -0
- package/src/sessions/dump.ts +105 -0
- package/src/sessions/fork.ts +127 -0
- package/src/sessions/handlers.ts +158 -0
- package/src/sessions/harness.ts +167 -0
- package/src/sessions/index.ts +26 -0
- package/src/sessions/last-live.ts +111 -0
- package/src/sessions/processes.ts +413 -0
- package/src/sessions/registry.ts +785 -0
- package/src/sessions/search.ts +278 -0
- package/src/sessions/status.ts +209 -0
- package/src/sessions/terminals.ts +72 -0
- package/src/sessions/workspace.ts +140 -0
- package/src/topics/handlers.ts +42 -0
- package/src/topics/index.ts +2 -0
- package/src/topics/topics.ts +290 -0
- package/src/transcript/files.ts +201 -0
- package/src/transcript/fold.ts +833 -0
- package/src/transcript/index.ts +16 -0
- package/src/transcript/read.ts +82 -0
- package/src/transcript/tail.ts +195 -0
- package/src/transcript/transcripts.ts +162 -0
- package/src/translate/helper.ts +87 -0
- package/src/translate/index.ts +2 -0
- package/src/translate/translate.ts +127 -0
- package/src/transport/conn.ts +129 -0
- package/src/transport/dial.ts +65 -0
- package/src/transport/driver.ts +102 -0
- package/src/transport/entry.ts +39 -0
- package/src/transport/framing.ts +131 -0
- package/src/transport/index.ts +8 -0
- package/src/transport/listener.ts +39 -0
- package/src/transport/uds.ts +88 -0
- package/src/transport/ws.ts +170 -0
- package/src/upstream/events.ts +125 -0
- package/src/upstream/gateway.ts +275 -0
- package/src/upstream/index.ts +8 -0
- package/src/upstream/json.ts +81 -0
- package/src/upstream/requests.ts +234 -0
- package/src/upstream/stats.ts +99 -0
- package/src/upstream/status.ts +281 -0
- package/src/upstream/usage.ts +208 -0
- package/src/upstream/webhook.ts +141 -0
- package/src/version.ts +8 -0
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
export {
|
|
2
|
+
type AgentNames,
|
|
3
|
+
type TranscriptFile,
|
|
4
|
+
TranscriptFiles,
|
|
5
|
+
type TranscriptFilesDeps,
|
|
6
|
+
} from "./files.ts";
|
|
7
|
+
export {
|
|
8
|
+
NO_FACTS,
|
|
9
|
+
readRecord,
|
|
10
|
+
type TranscriptFacts,
|
|
11
|
+
TranscriptFold,
|
|
12
|
+
type TranscriptRecord,
|
|
13
|
+
} from "./fold.ts";
|
|
14
|
+
export { READ_LIMIT, readSlice } from "./read.ts";
|
|
15
|
+
export { type Appended, FOLD_TAIL_BYTES, TranscriptTail } from "./tail.ts";
|
|
16
|
+
export { Transcripts, type TranscriptsDeps } from "./transcripts.ts";
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import { closeSync, openSync, readSync, statSync } from "node:fs";
|
|
2
|
+
import type { TranscriptReadResult, Sid } from "@ccmsg/protocol";
|
|
3
|
+
import { OpError } from "../dispatch/index.ts";
|
|
4
|
+
|
|
5
|
+
/** How much of a transcript one read may carry.
|
|
6
|
+
*
|
|
7
|
+
* The contract lets a caller ask for less and says the instance narrows the
|
|
8
|
+
* ask to its own limit, so this is both the ceiling and the default: a caller
|
|
9
|
+
* that states nothing gets a slice of the size the paging was designed around,
|
|
10
|
+
* and one that asks for a file's worth gets the same. It is the file-read
|
|
11
|
+
* limit, for the same reason — one connection, one bounded payload. */
|
|
12
|
+
export const READ_LIMIT = 512 * 1024;
|
|
13
|
+
|
|
14
|
+
/** A slice of a transcript, read backwards from an offset.
|
|
15
|
+
*
|
|
16
|
+
* Paging is by byte offset aligned to line boundaries (§3.3): the end of the
|
|
17
|
+
* file is read first, and each further page asks for what began before the
|
|
18
|
+
* slice just read. Nothing is scanned whole and no index is built, which is
|
|
19
|
+
* what lets a transcript of any size be read from its end.
|
|
20
|
+
*
|
|
21
|
+
* The offsets are the ones the `transcript` topic's frames carry, so what a
|
|
22
|
+
* client reads and what arrives live stitch together without overlap. */
|
|
23
|
+
export function readSlice(
|
|
24
|
+
sid: Sid,
|
|
25
|
+
file: string,
|
|
26
|
+
before?: number,
|
|
27
|
+
maxBytes?: number,
|
|
28
|
+
): TranscriptReadResult {
|
|
29
|
+
let size: number;
|
|
30
|
+
try {
|
|
31
|
+
size = statSync(file).size;
|
|
32
|
+
} catch {
|
|
33
|
+
throw new OpError("not_found", `the transcript of ${sid} could not be read`);
|
|
34
|
+
}
|
|
35
|
+
const until = before === undefined ? size : Math.min(before, size);
|
|
36
|
+
const want = Math.min(maxBytes ?? READ_LIMIT, READ_LIMIT);
|
|
37
|
+
const from = Math.max(0, until - want);
|
|
38
|
+
// One byte before what was asked for, so the read can see whether it began
|
|
39
|
+
// on a line boundary. Without it a slice starting exactly at a record's
|
|
40
|
+
// first byte is indistinguishable from one starting inside the record
|
|
41
|
+
// before it, and the whole first record would be dropped as a fragment.
|
|
42
|
+
const probe = from === 0 ? 0 : from - 1;
|
|
43
|
+
// The offsets are found in the bytes and never in decoded text. The probe
|
|
44
|
+
// byte, and a slice that begins part-way into a file, both land wherever the
|
|
45
|
+
// arithmetic puts them — inside a character as readily as before one — and
|
|
46
|
+
// decoding first would turn those bytes into replacement characters of a
|
|
47
|
+
// different length, moving every offset derived from them.
|
|
48
|
+
const bytes = slice(file, probe, until);
|
|
49
|
+
// What of the read is whole records: everything up to the last newline. A
|
|
50
|
+
// record the writer has not finished ends the file without one.
|
|
51
|
+
const lastNewline = bytes.lastIndexOf(NEWLINE);
|
|
52
|
+
if (lastNewline < 0) return { sid, lines: [], start: until, end: until, size };
|
|
53
|
+
const complete = bytes.subarray(0, lastNewline + 1);
|
|
54
|
+
const end = probe + complete.byteLength;
|
|
55
|
+
// Whatever of a record preceded `from` is dropped: empty when `from` already
|
|
56
|
+
// sat on a boundary, and the tail of a record the caller has already read
|
|
57
|
+
// otherwise. Either way the slice starts after the newline that ended it.
|
|
58
|
+
const first = probe < from ? complete.indexOf(NEWLINE) + 1 : 0;
|
|
59
|
+
const lines: string[] = [];
|
|
60
|
+
for (let at = first; at < complete.byteLength;) {
|
|
61
|
+
const newline = complete.indexOf(NEWLINE, at);
|
|
62
|
+
lines.push(complete.toString("utf8", at, newline));
|
|
63
|
+
at = newline + 1;
|
|
64
|
+
}
|
|
65
|
+
return { sid, lines, start: probe + first, end, size };
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
const NEWLINE = 0x0a;
|
|
69
|
+
|
|
70
|
+
/** The bytes in a range. A range that reads short — the file was truncated
|
|
71
|
+
* between the stat and the read — yields what was actually there. */
|
|
72
|
+
function slice(file: string, from: number, to: number): Buffer {
|
|
73
|
+
if (to <= from) return Buffer.alloc(0);
|
|
74
|
+
const handle = openSync(file, "r");
|
|
75
|
+
try {
|
|
76
|
+
const buffer = Buffer.alloc(to - from);
|
|
77
|
+
const read = readSync(handle, buffer, 0, buffer.length, from);
|
|
78
|
+
return buffer.subarray(0, read);
|
|
79
|
+
} finally {
|
|
80
|
+
closeSync(handle);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
import { type FSWatcher, statSync, watch } from "node:fs";
|
|
2
|
+
import { open, stat } from "node:fs/promises";
|
|
3
|
+
import { CONFIRM_POLL_MS } from "../sessions/harness.ts";
|
|
4
|
+
|
|
5
|
+
/** How much of an existing transcript is read when a tail starts.
|
|
6
|
+
*
|
|
7
|
+
* The fold's two values both describe the present — the error the latest turn
|
|
8
|
+
* ended on, and the last time a person spoke — so what a tail needs on opening
|
|
9
|
+
* is the recent end of the file, not its history. A megabyte is a few hundred
|
|
10
|
+
* records at the sizes the harness writes, which reaches back past the current
|
|
11
|
+
* turn by a wide margin while costing one read of fixed size however large the
|
|
12
|
+
* file has grown (§3.3: a transcript of any size is read from its end).
|
|
13
|
+
*
|
|
14
|
+
* A person who has not spoken within it is reported as having no known input
|
|
15
|
+
* rather than as having spoken long ago, which is what the contract's absent
|
|
16
|
+
* `last_user_input_at` already means. */
|
|
17
|
+
export const FOLD_TAIL_BYTES = 1024 * 1024;
|
|
18
|
+
|
|
19
|
+
/** What the tail found appended, with the offsets that place it.
|
|
20
|
+
*
|
|
21
|
+
* The offsets are the ones a transcript read pages by, so what arrives live
|
|
22
|
+
* and what was read stitch together without reading anything twice. `end` is
|
|
23
|
+
* past the last complete line, which is not `size` when a record is still
|
|
24
|
+
* being written. */
|
|
25
|
+
export interface Appended {
|
|
26
|
+
readonly lines: readonly string[];
|
|
27
|
+
readonly start: number;
|
|
28
|
+
readonly end: number;
|
|
29
|
+
readonly size: number;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export interface TailDeps {
|
|
33
|
+
/** Complete lines only; a record still being written waits for its end. */
|
|
34
|
+
readonly onAppended: (appended: Appended) => void;
|
|
35
|
+
/** The end of the file as it stood when the tail opened, oldest first. The
|
|
36
|
+
* seed of the fold, not something a subscriber is sent. */
|
|
37
|
+
readonly onSeed: (lines: readonly string[]) => void;
|
|
38
|
+
/** The file is not the one the tail was reading: it shrank, so what was
|
|
39
|
+
* folded out of the old contents no longer describes it. */
|
|
40
|
+
readonly onTruncated: () => void;
|
|
41
|
+
readonly pollMs?: number;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/** One transcript file, followed while somebody wants it (§6.3).
|
|
45
|
+
*
|
|
46
|
+
* Watch plus a low-rate confirmation poll, for the reason and at the interval
|
|
47
|
+
* the sessions directory uses (§5.1): the watch is the route and the poll is
|
|
48
|
+
* the backstop for what a delayed FSEvents queue is still sitting on. The
|
|
49
|
+
* interval is shared rather than chosen again, so the two watches cannot
|
|
50
|
+
* drift into two different answers to the same question. */
|
|
51
|
+
export class TranscriptTail {
|
|
52
|
+
#watcher: FSWatcher | undefined;
|
|
53
|
+
#timer: ReturnType<typeof setInterval> | undefined;
|
|
54
|
+
#reading: Promise<void> = Promise.resolve();
|
|
55
|
+
/** Just past the last complete line consumed. A record still being written
|
|
56
|
+
* leaves the offset before it, so the next read takes it whole rather than
|
|
57
|
+
* having to hold half of it — which also keeps a character split across two
|
|
58
|
+
* writes from being decoded in halves. */
|
|
59
|
+
#offset = 0;
|
|
60
|
+
#size = 0;
|
|
61
|
+
|
|
62
|
+
constructor(
|
|
63
|
+
private readonly path: string,
|
|
64
|
+
private readonly deps: TailDeps,
|
|
65
|
+
) {
|
|
66
|
+
// Where the file ends, read before anything can ask. A subscription's
|
|
67
|
+
// snapshot states this and is answered in the same turn the tail is
|
|
68
|
+
// created, so a size that only the awaited seed had filled in would be
|
|
69
|
+
// reported as zero and every byte already written would look appended.
|
|
70
|
+
// Reading it here also fixes what the seed reads: the seed takes this size
|
|
71
|
+
// rather than stating a newer one, so nothing lands between the size the
|
|
72
|
+
// subscriber was given and the first frame it is sent.
|
|
73
|
+
this.#size = sizeNow(path);
|
|
74
|
+
this.#offset = this.#size;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** The transcript's size as last observed, which is what a subscription's
|
|
78
|
+
* snapshot states and where the frames after it begin. */
|
|
79
|
+
get size(): number {
|
|
80
|
+
return this.#size;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
get running(): boolean {
|
|
84
|
+
return this.#watcher !== undefined || this.#timer !== undefined;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/** Begin following, seeding the fold from the end of what is already there.
|
|
88
|
+
* Resolves once the seed has been read, so a snapshot taken after it states
|
|
89
|
+
* a size the fold has caught up with. */
|
|
90
|
+
async start(): Promise<void> {
|
|
91
|
+
if (this.running) return;
|
|
92
|
+
await this.#seed();
|
|
93
|
+
try {
|
|
94
|
+
this.#watcher = watch(this.path, () => void this.refresh());
|
|
95
|
+
} catch {
|
|
96
|
+
// The file does not exist yet — a session that has not been written to.
|
|
97
|
+
// The poll covers the wait and picks it up when it appears.
|
|
98
|
+
this.#watcher = undefined;
|
|
99
|
+
}
|
|
100
|
+
this.#timer = setInterval(() => void this.refresh(), this.deps.pollMs ?? CONFIRM_POLL_MS);
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
stop(): void {
|
|
104
|
+
this.#watcher?.close();
|
|
105
|
+
this.#watcher = undefined;
|
|
106
|
+
if (this.#timer !== undefined) clearInterval(this.#timer);
|
|
107
|
+
this.#timer = undefined;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/** Read what has been appended since the last read. Reads are chained rather
|
|
111
|
+
* than overlapped, so a watch event and a poll arriving together cannot
|
|
112
|
+
* interleave their reads of the same file. */
|
|
113
|
+
refresh(): Promise<void> {
|
|
114
|
+
this.#reading = this.#reading.then(() => this.#read());
|
|
115
|
+
return this.#reading;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
async #seed(): Promise<void> {
|
|
119
|
+
const size = this.#size;
|
|
120
|
+
if (size === 0) return;
|
|
121
|
+
const from = Math.max(0, size - FOLD_TAIL_BYTES);
|
|
122
|
+
const text = await this.#slice(from, size);
|
|
123
|
+
const complete = whole(text);
|
|
124
|
+
this.#offset = from + byteLength(complete);
|
|
125
|
+
const lines = split(complete);
|
|
126
|
+
// The first line is half a record whenever the read began mid-file, so it
|
|
127
|
+
// is dropped: what the fold reads are whole records or nothing.
|
|
128
|
+
if (from > 0) lines.shift();
|
|
129
|
+
this.deps.onSeed(lines);
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
async #read(): Promise<void> {
|
|
133
|
+
const size = await this.#stat();
|
|
134
|
+
if (size < this.#offset) {
|
|
135
|
+
// Shorter than what was already consumed: the file was replaced, so what
|
|
136
|
+
// was folded out of it describes nothing, and reading resumes from its
|
|
137
|
+
// beginning.
|
|
138
|
+
this.#offset = 0;
|
|
139
|
+
this.deps.onTruncated();
|
|
140
|
+
}
|
|
141
|
+
this.#size = size;
|
|
142
|
+
if (size === this.#offset) return;
|
|
143
|
+
const complete = whole(await this.#slice(this.#offset, size));
|
|
144
|
+
if (complete.length === 0) return;
|
|
145
|
+
const start = this.#offset;
|
|
146
|
+
const end = start + byteLength(complete);
|
|
147
|
+
this.#offset = end;
|
|
148
|
+
this.deps.onAppended({ lines: split(complete), start, end, size });
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
async #stat(): Promise<number> {
|
|
152
|
+
try {
|
|
153
|
+
return (await stat(this.path)).size;
|
|
154
|
+
} catch {
|
|
155
|
+
// Not there. Nothing was appended, and the poll keeps looking.
|
|
156
|
+
return this.#offset;
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/** The bytes in a range, as text. A range that reads short — the file was
|
|
161
|
+
* truncated between the stat and the read — yields what was actually there. */
|
|
162
|
+
async #slice(from: number, to: number): Promise<string> {
|
|
163
|
+
const handle = await open(this.path, "r").catch(() => undefined);
|
|
164
|
+
if (handle === undefined) return "";
|
|
165
|
+
try {
|
|
166
|
+
const buffer = Buffer.alloc(to - from);
|
|
167
|
+
const { bytesRead } = await handle.read(buffer, 0, buffer.length, from);
|
|
168
|
+
return buffer.subarray(0, bytesRead).toString("utf8");
|
|
169
|
+
} finally {
|
|
170
|
+
await handle.close();
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
/** What of a read is whole records: everything up to and including the last
|
|
176
|
+
* newline. A transcript ends every record with one, so what follows the last
|
|
177
|
+
* is a record the writer has not finished. */
|
|
178
|
+
function whole(text: string): string {
|
|
179
|
+
const last = text.lastIndexOf("\n");
|
|
180
|
+
return last < 0 ? "" : text.slice(0, last + 1);
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
function split(complete: string): string[] {
|
|
184
|
+
return complete.split("\n").slice(0, -1);
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
function byteLength(text: string): number {
|
|
188
|
+
return Buffer.byteLength(text, "utf8");
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
/** How large the file is right now, or zero for one that is not there yet.
|
|
192
|
+
* Synchronous because the value is wanted before the first await. */
|
|
193
|
+
function sizeNow(path: string): number {
|
|
194
|
+
return statSync(path, { throwIfNoEntry: false })?.size ?? 0;
|
|
195
|
+
}
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
import type { InstanceId, Sid } from "@ccmsg/protocol";
|
|
2
|
+
import { topicParam, type TopicValue, type UpstreamResource } from "../topics/index.ts";
|
|
3
|
+
import { NO_FACTS, type TranscriptFacts, TranscriptFold } from "./fold.ts";
|
|
4
|
+
import { type Appended, TranscriptTail } from "./tail.ts";
|
|
5
|
+
|
|
6
|
+
export interface TranscriptsDeps {
|
|
7
|
+
readonly self: InstanceId;
|
|
8
|
+
/** Where a session's transcript is, as the session announced it (§5.1). A
|
|
9
|
+
* sid with no path is one that never said, and nothing is guessed for it. */
|
|
10
|
+
readonly pathOf: (sid: Sid) => string | undefined;
|
|
11
|
+
/** The one way a value reaches subscribers (§6.1). */
|
|
12
|
+
readonly publish: (topic: string, data: unknown) => void;
|
|
13
|
+
/** The fold now says something different about this session. What the fold
|
|
14
|
+
* settles is an input to the sessions domain (§5.1), so the domain that
|
|
15
|
+
* states those values is told to state them again. */
|
|
16
|
+
readonly onFacts: (sid: Sid) => void;
|
|
17
|
+
/** Overrides the confirmation poll, for a test that cannot wait. */
|
|
18
|
+
readonly pollMs?: number;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/** One tail and one fold per session, and the `transcript:<sid>` topic they
|
|
22
|
+
* feed (§3.3).
|
|
23
|
+
*
|
|
24
|
+
* The fold is one per session, not one per consumer: a line is read once and
|
|
25
|
+
* every value it settles is settled from that read, so the api error, the last
|
|
26
|
+
* human input and the appended bytes are three uses of one pass rather than
|
|
27
|
+
* three passes (M5).
|
|
28
|
+
*
|
|
29
|
+
* A tail runs while something wants it and stops when nothing does (§6.3).
|
|
30
|
+
* Subscription is one such want; a `hold` is the other, for the values the
|
|
31
|
+
* sessions domain states about a session nobody is watching the transcript of.
|
|
32
|
+
* They are counted together, so the last one to go is what stops the tail. */
|
|
33
|
+
export class Transcripts implements UpstreamResource {
|
|
34
|
+
readonly #followed = new Map<Sid, Followed>();
|
|
35
|
+
|
|
36
|
+
constructor(private readonly deps: TranscriptsDeps) {}
|
|
37
|
+
|
|
38
|
+
// --- UpstreamResource (§6.3)
|
|
39
|
+
|
|
40
|
+
start(topic: string): void {
|
|
41
|
+
const sid = topicParam(topic);
|
|
42
|
+
if (sid !== undefined) this.hold(sid);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
stop(topic: string): void {
|
|
46
|
+
const sid = topicParam(topic);
|
|
47
|
+
if (sid !== undefined) this.release(sid);
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/** Where the transcript ends as the subscription begins. What follows starts
|
|
51
|
+
* there, which is the whole of the snapshot for a topic whose frames are an
|
|
52
|
+
* append rather than a value (§6.2). A session whose transcript this
|
|
53
|
+
* instance cannot find has nothing to state, and the subscriber begins at
|
|
54
|
+
* the first thing appended after one appears. */
|
|
55
|
+
snapshot(topic: string): readonly TopicValue[] {
|
|
56
|
+
const sid = topicParam(topic);
|
|
57
|
+
const followed = sid === undefined ? undefined : this.#followed.get(sid);
|
|
58
|
+
if (sid === undefined || followed === undefined) return [];
|
|
59
|
+
return [{ instance: this.deps.self, data: { sid, size: followed.tail.size } }];
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/** What the fold currently says about a session. Empty for one not being
|
|
63
|
+
* followed, which is the same as a transcript that has said nothing. */
|
|
64
|
+
facts(sid: Sid): TranscriptFacts {
|
|
65
|
+
return this.#followed.get(sid)?.fold.facts ?? NO_FACTS;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/** Whether a session's transcript is being followed, which is how "the
|
|
69
|
+
* subscription drives the resource" is observable from outside. */
|
|
70
|
+
following(sid: Sid): boolean {
|
|
71
|
+
return this.#followed.get(sid)?.tail.running === true;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/** Ask for a session's transcript to be followed. Each hold is released
|
|
75
|
+
* once; the tail runs until the last is. */
|
|
76
|
+
hold(sid: Sid): void {
|
|
77
|
+
const held = this.#followed.get(sid);
|
|
78
|
+
if (held !== undefined) {
|
|
79
|
+
held.holds += 1;
|
|
80
|
+
return;
|
|
81
|
+
}
|
|
82
|
+
const path = this.deps.pathOf(sid);
|
|
83
|
+
if (path === undefined) return;
|
|
84
|
+
const followed = this.#follow(sid, path);
|
|
85
|
+
this.#followed.set(sid, followed);
|
|
86
|
+
void followed.tail.start();
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
release(sid: Sid): void {
|
|
90
|
+
const held = this.#followed.get(sid);
|
|
91
|
+
if (held === undefined) return;
|
|
92
|
+
held.holds -= 1;
|
|
93
|
+
if (held.holds > 0) return;
|
|
94
|
+
this.#followed.delete(sid);
|
|
95
|
+
held.tail.stop();
|
|
96
|
+
// What the fold held goes with it: the values it derived describe a file
|
|
97
|
+
// this instance is no longer reading, and stating them from memory would
|
|
98
|
+
// outlive the reading that justified them.
|
|
99
|
+
this.deps.onFacts(sid);
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/** Stop following everything. What shutdown reaches through the topics it
|
|
103
|
+
* drops; a hold taken outside a subscription needs the same door. */
|
|
104
|
+
stopAll(): void {
|
|
105
|
+
const followed = new Map(this.#followed);
|
|
106
|
+
this.#followed.clear();
|
|
107
|
+
for (const [sid, entry] of followed) {
|
|
108
|
+
entry.tail.stop();
|
|
109
|
+
this.deps.onFacts(sid);
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
#follow(sid: Sid, path: string): Followed {
|
|
114
|
+
const fold = new TranscriptFold();
|
|
115
|
+
const followed: Followed = {
|
|
116
|
+
holds: 1,
|
|
117
|
+
fold,
|
|
118
|
+
tail: new TranscriptTail(path, {
|
|
119
|
+
onSeed: (lines) => {
|
|
120
|
+
// The end of the file as it already stood: it settles what the fold
|
|
121
|
+
// says, and it is not an append, so nothing is published for it.
|
|
122
|
+
if (foldAll(fold, lines)) this.deps.onFacts(sid);
|
|
123
|
+
},
|
|
124
|
+
onAppended: (appended) => this.#appended(sid, fold, appended),
|
|
125
|
+
onTruncated: () => {
|
|
126
|
+
fold.reset();
|
|
127
|
+
this.deps.onFacts(sid);
|
|
128
|
+
},
|
|
129
|
+
...(this.deps.pollMs === undefined ? {} : { pollMs: this.deps.pollMs }),
|
|
130
|
+
}),
|
|
131
|
+
};
|
|
132
|
+
return followed;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/** The one pass over what was appended: the lines go to the fold and to the
|
|
136
|
+
* topic, in that order, and are not read a second time for either (M5). */
|
|
137
|
+
#appended(sid: Sid, fold: TranscriptFold, appended: Appended): void {
|
|
138
|
+
const changed = foldAll(fold, appended.lines);
|
|
139
|
+
this.deps.publish(`transcript:${sid}`, {
|
|
140
|
+
sid,
|
|
141
|
+
lines: [...appended.lines],
|
|
142
|
+
start: appended.start,
|
|
143
|
+
end: appended.end,
|
|
144
|
+
size: appended.size,
|
|
145
|
+
});
|
|
146
|
+
if (changed) this.deps.onFacts(sid);
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
interface Followed {
|
|
151
|
+
holds: number;
|
|
152
|
+
readonly fold: TranscriptFold;
|
|
153
|
+
readonly tail: TranscriptTail;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
function foldAll(fold: TranscriptFold, lines: readonly string[]): boolean {
|
|
157
|
+
let changed = false;
|
|
158
|
+
for (const line of lines) {
|
|
159
|
+
if (fold.line(line)) changed = true;
|
|
160
|
+
}
|
|
161
|
+
return changed;
|
|
162
|
+
}
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
import { accessSync, constants } from "node:fs";
|
|
2
|
+
import { isAbsolute } from "node:path";
|
|
3
|
+
import type { Capability } from "@ccmsg/protocol";
|
|
4
|
+
import { ConfigError, type UpstreamConfig } from "../instance/config.ts";
|
|
5
|
+
|
|
6
|
+
/** One exchange with the helper: a line in, a line out.
|
|
7
|
+
*
|
|
8
|
+
* The helper is a process that reads one JSON line and answers with one, and
|
|
9
|
+
* this is that process as everything above it needs to see it. Named rather
|
|
10
|
+
* than reached through directly so the wire can be exercised without a program
|
|
11
|
+
* on the host answering it. */
|
|
12
|
+
export interface HelperChannel {
|
|
13
|
+
write(line: string): Promise<void>;
|
|
14
|
+
/** The next line the helper wrote, or `undefined` once it has written its
|
|
15
|
+
* last — which is how a helper that died is told from one that is thinking. */
|
|
16
|
+
read(): Promise<string | undefined>;
|
|
17
|
+
kill(): void;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/** Read the helper the config names (§8.2).
|
|
21
|
+
*
|
|
22
|
+
* A helper that is named and cannot be run ends the start rather than leaving
|
|
23
|
+
* translation silently off (DV-Q9): an instance without the capability looks
|
|
24
|
+
* exactly like one nobody configured, and the operator who named a program
|
|
25
|
+
* meant to have it. */
|
|
26
|
+
export function translateSetup(config: UpstreamConfig, file: string): string | undefined {
|
|
27
|
+
const helper = config.translate_helper;
|
|
28
|
+
if (helper === undefined) return undefined;
|
|
29
|
+
if (!isAbsolute(helper)) {
|
|
30
|
+
throw new ConfigError(file, "upstream.translate_helper must be an absolute path");
|
|
31
|
+
}
|
|
32
|
+
try {
|
|
33
|
+
accessSync(helper, constants.X_OK);
|
|
34
|
+
} catch (cause) {
|
|
35
|
+
throw new ConfigError(
|
|
36
|
+
file,
|
|
37
|
+
`upstream.translate_helper ${helper} cannot be run (${String(cause)})`,
|
|
38
|
+
);
|
|
39
|
+
}
|
|
40
|
+
return helper;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/** Present exactly where a helper is configured: translation happens on this
|
|
44
|
+
* host or not at all, and a client is told which before it asks. */
|
|
45
|
+
export function translateCapabilities(helper?: string): Capability[] {
|
|
46
|
+
return helper === undefined ? [] : ["translate"];
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** Start the configured helper, speaking one JSON line at a time. */
|
|
50
|
+
export function spawnHelper(path: string): HelperChannel {
|
|
51
|
+
const child = Bun.spawn([path], { stdin: "pipe", stdout: "pipe", stderr: "inherit" });
|
|
52
|
+
const stdin = child.stdin;
|
|
53
|
+
const lines = readLines(child.stdout);
|
|
54
|
+
return {
|
|
55
|
+
write: async (line) => {
|
|
56
|
+
// The write is awaited because a full pipe answers with a promise rather
|
|
57
|
+
// than a count, and a line half-taken is a batch the helper never sees.
|
|
58
|
+
await stdin.write(line);
|
|
59
|
+
await stdin.flush();
|
|
60
|
+
},
|
|
61
|
+
read: () => lines(),
|
|
62
|
+
kill: () => {
|
|
63
|
+
child.kill();
|
|
64
|
+
},
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/** The helper's output, one line at a time. */
|
|
69
|
+
function readLines(stream: ReadableStream<Uint8Array>): () => Promise<string | undefined> {
|
|
70
|
+
const reader = stream.getReader();
|
|
71
|
+
const decoder = new TextDecoder();
|
|
72
|
+
let buffer = "";
|
|
73
|
+
return async () => {
|
|
74
|
+
for (;;) {
|
|
75
|
+
const at = buffer.indexOf("\n");
|
|
76
|
+
if (at >= 0) {
|
|
77
|
+
const line = buffer.slice(0, at);
|
|
78
|
+
buffer = buffer.slice(at + 1);
|
|
79
|
+
if (line.trim() !== "") return line;
|
|
80
|
+
continue;
|
|
81
|
+
}
|
|
82
|
+
const { done, value } = await reader.read();
|
|
83
|
+
if (done) return undefined;
|
|
84
|
+
buffer += decoder.decode(value, { stream: true });
|
|
85
|
+
}
|
|
86
|
+
};
|
|
87
|
+
}
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
import type { TranslateResult, TranslateRunArgs, TranslateRunResult } from "@ccmsg/protocol";
|
|
2
|
+
import { type HandlerInput, OpError } from "../dispatch/index.ts";
|
|
3
|
+
import { type HelperChannel, spawnHelper } from "./helper.ts";
|
|
4
|
+
|
|
5
|
+
/** How long one batch may take before the helper is assumed wedged.
|
|
6
|
+
*
|
|
7
|
+
* The deadline grows with the input because the work does: a sentence comes
|
|
8
|
+
* back in seconds and a page takes minutes. The helper reports no progress, so
|
|
9
|
+
* a batch that outlives its deadline can only be ended by ending the helper —
|
|
10
|
+
* the next call starts a fresh one. */
|
|
11
|
+
const BASE_MS = 10_000;
|
|
12
|
+
const PER_100_CHARS_MS = 1_000;
|
|
13
|
+
const MAX_MS = 120_000;
|
|
14
|
+
|
|
15
|
+
export function deadlineMs(chars: number): number {
|
|
16
|
+
return Math.min(MAX_MS, BASE_MS + Math.ceil(chars / 100) * PER_100_CHARS_MS);
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export interface TranslateDeps {
|
|
20
|
+
/** Starts the helper. Replaced in tests, which answer the same line protocol
|
|
21
|
+
* without a program on the host. */
|
|
22
|
+
readonly start?: () => HelperChannel;
|
|
23
|
+
readonly deadlineMs?: (chars: number) => number;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/** The host's translator, kept running between calls.
|
|
27
|
+
*
|
|
28
|
+
* Resident because starting it is the expensive part (DR-0023 §3.1): the
|
|
29
|
+
* process loads a translation session once and answers from it, so a batch per
|
|
30
|
+
* process would pay that cost on every keystroke's worth of text. One batch is
|
|
31
|
+
* in flight at a time — the helper answers one line per line it is given, and
|
|
32
|
+
* two batches sharing that channel could not tell the answers apart. */
|
|
33
|
+
export class Translate {
|
|
34
|
+
#helper: HelperChannel | undefined;
|
|
35
|
+
#queue: Promise<unknown> = Promise.resolve();
|
|
36
|
+
#next = 0;
|
|
37
|
+
|
|
38
|
+
constructor(
|
|
39
|
+
private readonly helperPath: string,
|
|
40
|
+
private readonly deps: TranslateDeps = {},
|
|
41
|
+
) {}
|
|
42
|
+
|
|
43
|
+
async run(args: TranslateRunArgs): Promise<TranslateRunResult> {
|
|
44
|
+
// Nothing to translate needs no helper, and starting one to answer with an
|
|
45
|
+
// empty list would make the empty batch the probe it is not.
|
|
46
|
+
if (args.texts.length === 0) return { results: [] };
|
|
47
|
+
const done = this.#queue.then(
|
|
48
|
+
() => this.#exchange(args.texts),
|
|
49
|
+
() => this.#exchange(args.texts),
|
|
50
|
+
);
|
|
51
|
+
this.#queue = done.catch(() => undefined);
|
|
52
|
+
return { results: await done };
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** Stop the helper. What shutdown calls, and what a wedged batch does before
|
|
56
|
+
* it gives up. */
|
|
57
|
+
stop(): void {
|
|
58
|
+
this.#helper?.kill();
|
|
59
|
+
this.#helper = undefined;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
async #exchange(texts: readonly string[]): Promise<TranslateResult[]> {
|
|
63
|
+
const helper = (this.#helper ??= (this.deps.start ?? (() => spawnHelper(this.helperPath)))());
|
|
64
|
+
const id = `${(this.#next += 1)}`;
|
|
65
|
+
const chars = texts.reduce((total, text) => total + text.length, 0);
|
|
66
|
+
const budget = (this.deps.deadlineMs ?? deadlineMs)(chars);
|
|
67
|
+
let expired: ReturnType<typeof setTimeout> | undefined;
|
|
68
|
+
try {
|
|
69
|
+
await helper.write(`${JSON.stringify({ id, texts })}\n`);
|
|
70
|
+
const line = await Promise.race([
|
|
71
|
+
helper.read(),
|
|
72
|
+
new Promise<never>((_resolve, reject) => {
|
|
73
|
+
expired = setTimeout(() => {
|
|
74
|
+
reject(new Error(`the helper did not answer within ${budget}ms`));
|
|
75
|
+
}, budget);
|
|
76
|
+
}),
|
|
77
|
+
]);
|
|
78
|
+
if (line === undefined) throw new Error("the helper stopped before it answered");
|
|
79
|
+
return read(line, id, texts.length);
|
|
80
|
+
} catch (cause) {
|
|
81
|
+
// Whatever went wrong, this helper is no longer known to be in step with
|
|
82
|
+
// the line protocol, so it is ended and the next call starts a fresh one.
|
|
83
|
+
this.stop();
|
|
84
|
+
throw new OpError(
|
|
85
|
+
"translate_helper_failed",
|
|
86
|
+
`the translation helper failed: ${String(cause)}`,
|
|
87
|
+
);
|
|
88
|
+
} finally {
|
|
89
|
+
if (expired !== undefined) clearTimeout(expired);
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/** One answer line, read as the batch it was meant to answer.
|
|
95
|
+
*
|
|
96
|
+
* The id and the count are both checked: an answer to another batch, or one
|
|
97
|
+
* holding a different number of results, would hand the caller texts that are
|
|
98
|
+
* not the ones it sent. */
|
|
99
|
+
function read(line: string, id: string, expected: number): TranslateResult[] {
|
|
100
|
+
const parsed: unknown = JSON.parse(line);
|
|
101
|
+
if (typeof parsed !== "object" || parsed === null) throw new Error("the answer is not an object");
|
|
102
|
+
const fields = parsed as Record<string, unknown>;
|
|
103
|
+
if (fields["id"] !== id) throw new Error(`the answer names batch ${String(fields["id"])}`);
|
|
104
|
+
const results = fields["results"];
|
|
105
|
+
if (!Array.isArray(results) || results.length !== expected) {
|
|
106
|
+
throw new Error(
|
|
107
|
+
`the answer holds ${Array.isArray(results) ? results.length : 0} of ${expected}`,
|
|
108
|
+
);
|
|
109
|
+
}
|
|
110
|
+
return results.map((entry): TranslateResult => {
|
|
111
|
+
const item = (typeof entry === "object" && entry !== null ? entry : {}) as Record<
|
|
112
|
+
string,
|
|
113
|
+
unknown
|
|
114
|
+
>;
|
|
115
|
+
const text = item["text"];
|
|
116
|
+
if (item["ok"] === true && typeof text === "string") return { ok: true, text };
|
|
117
|
+
const error = item["error"];
|
|
118
|
+
return { ok: false, error: typeof error === "string" ? error : "the helper stated no reason" };
|
|
119
|
+
});
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
export function translateHandlers(translate: Translate) {
|
|
123
|
+
return {
|
|
124
|
+
translate_run: (input: HandlerInput): Promise<TranslateRunResult> =>
|
|
125
|
+
translate.run(input.args as unknown as TranslateRunArgs),
|
|
126
|
+
};
|
|
127
|
+
}
|