@ccmsg/cli 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (102) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +23 -0
  3. package/package.json +32 -0
  4. package/src/cli.ts +1074 -0
  5. package/src/daemon/control.ts +88 -0
  6. package/src/daemon/index.ts +6 -0
  7. package/src/daemon/link.ts +93 -0
  8. package/src/daemon/log.ts +116 -0
  9. package/src/daemon/registry.ts +285 -0
  10. package/src/daemon/snapshot.ts +115 -0
  11. package/src/daemon/supervise.ts +446 -0
  12. package/src/dispatch/caller.ts +47 -0
  13. package/src/dispatch/dispatch.ts +128 -0
  14. package/src/dispatch/handler.ts +55 -0
  15. package/src/dispatch/identity.ts +22 -0
  16. package/src/dispatch/index.ts +5 -0
  17. package/src/dispatch/result.ts +58 -0
  18. package/src/files/containment.ts +263 -0
  19. package/src/files/files.ts +421 -0
  20. package/src/files/index.ts +14 -0
  21. package/src/files/sandbox.ts +0 -0
  22. package/src/greeting/hook.ts +48 -0
  23. package/src/greeting/index.ts +2 -0
  24. package/src/greeting/meta.ts +66 -0
  25. package/src/instance/config.ts +424 -0
  26. package/src/instance/handlers.ts +28 -0
  27. package/src/instance/identity.ts +44 -0
  28. package/src/instance/index.ts +8 -0
  29. package/src/instance/instance.ts +911 -0
  30. package/src/instance/lock.ts +108 -0
  31. package/src/instance/log.ts +30 -0
  32. package/src/instance/paths.ts +200 -0
  33. package/src/instance/socket.ts +62 -0
  34. package/src/kv/index.ts +2 -0
  35. package/src/kv/merge.ts +66 -0
  36. package/src/kv/store.ts +195 -0
  37. package/src/launcher/index.ts +4 -0
  38. package/src/launcher/launcher.ts +190 -0
  39. package/src/launcher/roots.ts +32 -0
  40. package/src/launcher/spawn.ts +81 -0
  41. package/src/launcher/tree.ts +80 -0
  42. package/src/mesh/index.ts +5 -0
  43. package/src/mesh/keys.ts +158 -0
  44. package/src/mesh/mesh.ts +1169 -0
  45. package/src/mesh/probe.ts +100 -0
  46. package/src/mesh/relay.ts +147 -0
  47. package/src/mesh/wire.ts +96 -0
  48. package/src/messaging/delivery.ts +375 -0
  49. package/src/messaging/direct.ts +433 -0
  50. package/src/messaging/handlers.ts +14 -0
  51. package/src/messaging/inbox.ts +191 -0
  52. package/src/messaging/index.ts +5 -0
  53. package/src/messaging/notify.ts +117 -0
  54. package/src/plugin/claude.ts +148 -0
  55. package/src/plugin/index.ts +13 -0
  56. package/src/plugin/install.ts +416 -0
  57. package/src/service/index.ts +1 -0
  58. package/src/service/service.ts +359 -0
  59. package/src/sessions/classify.ts +66 -0
  60. package/src/sessions/dump.ts +105 -0
  61. package/src/sessions/fork.ts +127 -0
  62. package/src/sessions/handlers.ts +158 -0
  63. package/src/sessions/harness.ts +167 -0
  64. package/src/sessions/index.ts +26 -0
  65. package/src/sessions/last-live.ts +111 -0
  66. package/src/sessions/processes.ts +413 -0
  67. package/src/sessions/registry.ts +785 -0
  68. package/src/sessions/search.ts +278 -0
  69. package/src/sessions/status.ts +209 -0
  70. package/src/sessions/terminals.ts +72 -0
  71. package/src/sessions/workspace.ts +140 -0
  72. package/src/topics/handlers.ts +42 -0
  73. package/src/topics/index.ts +2 -0
  74. package/src/topics/topics.ts +290 -0
  75. package/src/transcript/files.ts +201 -0
  76. package/src/transcript/fold.ts +833 -0
  77. package/src/transcript/index.ts +16 -0
  78. package/src/transcript/read.ts +82 -0
  79. package/src/transcript/tail.ts +195 -0
  80. package/src/transcript/transcripts.ts +162 -0
  81. package/src/translate/helper.ts +87 -0
  82. package/src/translate/index.ts +2 -0
  83. package/src/translate/translate.ts +127 -0
  84. package/src/transport/conn.ts +129 -0
  85. package/src/transport/dial.ts +65 -0
  86. package/src/transport/driver.ts +102 -0
  87. package/src/transport/entry.ts +39 -0
  88. package/src/transport/framing.ts +131 -0
  89. package/src/transport/index.ts +8 -0
  90. package/src/transport/listener.ts +39 -0
  91. package/src/transport/uds.ts +88 -0
  92. package/src/transport/ws.ts +170 -0
  93. package/src/upstream/events.ts +125 -0
  94. package/src/upstream/gateway.ts +275 -0
  95. package/src/upstream/index.ts +8 -0
  96. package/src/upstream/json.ts +81 -0
  97. package/src/upstream/requests.ts +234 -0
  98. package/src/upstream/stats.ts +99 -0
  99. package/src/upstream/status.ts +281 -0
  100. package/src/upstream/usage.ts +208 -0
  101. package/src/upstream/webhook.ts +141 -0
  102. package/src/version.ts +8 -0
@@ -0,0 +1,16 @@
1
+ export {
2
+ type AgentNames,
3
+ type TranscriptFile,
4
+ TranscriptFiles,
5
+ type TranscriptFilesDeps,
6
+ } from "./files.ts";
7
+ export {
8
+ NO_FACTS,
9
+ readRecord,
10
+ type TranscriptFacts,
11
+ TranscriptFold,
12
+ type TranscriptRecord,
13
+ } from "./fold.ts";
14
+ export { READ_LIMIT, readSlice } from "./read.ts";
15
+ export { type Appended, FOLD_TAIL_BYTES, TranscriptTail } from "./tail.ts";
16
+ export { Transcripts, type TranscriptsDeps } from "./transcripts.ts";
@@ -0,0 +1,82 @@
1
+ import { closeSync, openSync, readSync, statSync } from "node:fs";
2
+ import type { TranscriptReadResult, Sid } from "@ccmsg/protocol";
3
+ import { OpError } from "../dispatch/index.ts";
4
+
5
+ /** How much of a transcript one read may carry.
6
+ *
7
+ * The contract lets a caller ask for less and says the instance narrows the
8
+ * ask to its own limit, so this is both the ceiling and the default: a caller
9
+ * that states nothing gets a slice of the size the paging was designed around,
10
+ * and one that asks for a file's worth gets the same. It is the file-read
11
+ * limit, for the same reason — one connection, one bounded payload. */
12
+ export const READ_LIMIT = 512 * 1024;
13
+
14
+ /** A slice of a transcript, read backwards from an offset.
15
+ *
16
+ * Paging is by byte offset aligned to line boundaries (§3.3): the end of the
17
+ * file is read first, and each further page asks for what began before the
18
+ * slice just read. Nothing is scanned whole and no index is built, which is
19
+ * what lets a transcript of any size be read from its end.
20
+ *
21
+ * The offsets are the ones the `transcript` topic's frames carry, so what a
22
+ * client reads and what arrives live stitch together without overlap. */
23
+ export function readSlice(
24
+ sid: Sid,
25
+ file: string,
26
+ before?: number,
27
+ maxBytes?: number,
28
+ ): TranscriptReadResult {
29
+ let size: number;
30
+ try {
31
+ size = statSync(file).size;
32
+ } catch {
33
+ throw new OpError("not_found", `the transcript of ${sid} could not be read`);
34
+ }
35
+ const until = before === undefined ? size : Math.min(before, size);
36
+ const want = Math.min(maxBytes ?? READ_LIMIT, READ_LIMIT);
37
+ const from = Math.max(0, until - want);
38
+ // One byte before what was asked for, so the read can see whether it began
39
+ // on a line boundary. Without it a slice starting exactly at a record's
40
+ // first byte is indistinguishable from one starting inside the record
41
+ // before it, and the whole first record would be dropped as a fragment.
42
+ const probe = from === 0 ? 0 : from - 1;
43
+ // The offsets are found in the bytes and never in decoded text. The probe
44
+ // byte, and a slice that begins part-way into a file, both land wherever the
45
+ // arithmetic puts them — inside a character as readily as before one — and
46
+ // decoding first would turn those bytes into replacement characters of a
47
+ // different length, moving every offset derived from them.
48
+ const bytes = slice(file, probe, until);
49
+ // What of the read is whole records: everything up to the last newline. A
50
+ // record the writer has not finished ends the file without one.
51
+ const lastNewline = bytes.lastIndexOf(NEWLINE);
52
+ if (lastNewline < 0) return { sid, lines: [], start: until, end: until, size };
53
+ const complete = bytes.subarray(0, lastNewline + 1);
54
+ const end = probe + complete.byteLength;
55
+ // Whatever of a record preceded `from` is dropped: empty when `from` already
56
+ // sat on a boundary, and the tail of a record the caller has already read
57
+ // otherwise. Either way the slice starts after the newline that ended it.
58
+ const first = probe < from ? complete.indexOf(NEWLINE) + 1 : 0;
59
+ const lines: string[] = [];
60
+ for (let at = first; at < complete.byteLength;) {
61
+ const newline = complete.indexOf(NEWLINE, at);
62
+ lines.push(complete.toString("utf8", at, newline));
63
+ at = newline + 1;
64
+ }
65
+ return { sid, lines, start: probe + first, end, size };
66
+ }
67
+
68
+ const NEWLINE = 0x0a;
69
+
70
+ /** The bytes in a range. A range that reads short — the file was truncated
71
+ * between the stat and the read — yields what was actually there. */
72
+ function slice(file: string, from: number, to: number): Buffer {
73
+ if (to <= from) return Buffer.alloc(0);
74
+ const handle = openSync(file, "r");
75
+ try {
76
+ const buffer = Buffer.alloc(to - from);
77
+ const read = readSync(handle, buffer, 0, buffer.length, from);
78
+ return buffer.subarray(0, read);
79
+ } finally {
80
+ closeSync(handle);
81
+ }
82
+ }
@@ -0,0 +1,195 @@
1
+ import { type FSWatcher, statSync, watch } from "node:fs";
2
+ import { open, stat } from "node:fs/promises";
3
+ import { CONFIRM_POLL_MS } from "../sessions/harness.ts";
4
+
5
+ /** How much of an existing transcript is read when a tail starts.
6
+ *
7
+ * The fold's two values both describe the present — the error the latest turn
8
+ * ended on, and the last time a person spoke — so what a tail needs on opening
9
+ * is the recent end of the file, not its history. A megabyte is a few hundred
10
+ * records at the sizes the harness writes, which reaches back past the current
11
+ * turn by a wide margin while costing one read of fixed size however large the
12
+ * file has grown (§3.3: a transcript of any size is read from its end).
13
+ *
14
+ * A person who has not spoken within it is reported as having no known input
15
+ * rather than as having spoken long ago, which is what the contract's absent
16
+ * `last_user_input_at` already means. */
17
+ export const FOLD_TAIL_BYTES = 1024 * 1024;
18
+
19
+ /** What the tail found appended, with the offsets that place it.
20
+ *
21
+ * The offsets are the ones a transcript read pages by, so what arrives live
22
+ * and what was read stitch together without reading anything twice. `end` is
23
+ * past the last complete line, which is not `size` when a record is still
24
+ * being written. */
25
+ export interface Appended {
26
+ readonly lines: readonly string[];
27
+ readonly start: number;
28
+ readonly end: number;
29
+ readonly size: number;
30
+ }
31
+
32
+ export interface TailDeps {
33
+ /** Complete lines only; a record still being written waits for its end. */
34
+ readonly onAppended: (appended: Appended) => void;
35
+ /** The end of the file as it stood when the tail opened, oldest first. The
36
+ * seed of the fold, not something a subscriber is sent. */
37
+ readonly onSeed: (lines: readonly string[]) => void;
38
+ /** The file is not the one the tail was reading: it shrank, so what was
39
+ * folded out of the old contents no longer describes it. */
40
+ readonly onTruncated: () => void;
41
+ readonly pollMs?: number;
42
+ }
43
+
44
+ /** One transcript file, followed while somebody wants it (§6.3).
45
+ *
46
+ * Watch plus a low-rate confirmation poll, for the reason and at the interval
47
+ * the sessions directory uses (§5.1): the watch is the route and the poll is
48
+ * the backstop for what a delayed FSEvents queue is still sitting on. The
49
+ * interval is shared rather than chosen again, so the two watches cannot
50
+ * drift into two different answers to the same question. */
51
+ export class TranscriptTail {
52
+ #watcher: FSWatcher | undefined;
53
+ #timer: ReturnType<typeof setInterval> | undefined;
54
+ #reading: Promise<void> = Promise.resolve();
55
+ /** Just past the last complete line consumed. A record still being written
56
+ * leaves the offset before it, so the next read takes it whole rather than
57
+ * having to hold half of it — which also keeps a character split across two
58
+ * writes from being decoded in halves. */
59
+ #offset = 0;
60
+ #size = 0;
61
+
62
+ constructor(
63
+ private readonly path: string,
64
+ private readonly deps: TailDeps,
65
+ ) {
66
+ // Where the file ends, read before anything can ask. A subscription's
67
+ // snapshot states this and is answered in the same turn the tail is
68
+ // created, so a size that only the awaited seed had filled in would be
69
+ // reported as zero and every byte already written would look appended.
70
+ // Reading it here also fixes what the seed reads: the seed takes this size
71
+ // rather than stating a newer one, so nothing lands between the size the
72
+ // subscriber was given and the first frame it is sent.
73
+ this.#size = sizeNow(path);
74
+ this.#offset = this.#size;
75
+ }
76
+
77
+ /** The transcript's size as last observed, which is what a subscription's
78
+ * snapshot states and where the frames after it begin. */
79
+ get size(): number {
80
+ return this.#size;
81
+ }
82
+
83
+ get running(): boolean {
84
+ return this.#watcher !== undefined || this.#timer !== undefined;
85
+ }
86
+
87
+ /** Begin following, seeding the fold from the end of what is already there.
88
+ * Resolves once the seed has been read, so a snapshot taken after it states
89
+ * a size the fold has caught up with. */
90
+ async start(): Promise<void> {
91
+ if (this.running) return;
92
+ await this.#seed();
93
+ try {
94
+ this.#watcher = watch(this.path, () => void this.refresh());
95
+ } catch {
96
+ // The file does not exist yet — a session that has not been written to.
97
+ // The poll covers the wait and picks it up when it appears.
98
+ this.#watcher = undefined;
99
+ }
100
+ this.#timer = setInterval(() => void this.refresh(), this.deps.pollMs ?? CONFIRM_POLL_MS);
101
+ }
102
+
103
+ stop(): void {
104
+ this.#watcher?.close();
105
+ this.#watcher = undefined;
106
+ if (this.#timer !== undefined) clearInterval(this.#timer);
107
+ this.#timer = undefined;
108
+ }
109
+
110
+ /** Read what has been appended since the last read. Reads are chained rather
111
+ * than overlapped, so a watch event and a poll arriving together cannot
112
+ * interleave their reads of the same file. */
113
+ refresh(): Promise<void> {
114
+ this.#reading = this.#reading.then(() => this.#read());
115
+ return this.#reading;
116
+ }
117
+
118
+ async #seed(): Promise<void> {
119
+ const size = this.#size;
120
+ if (size === 0) return;
121
+ const from = Math.max(0, size - FOLD_TAIL_BYTES);
122
+ const text = await this.#slice(from, size);
123
+ const complete = whole(text);
124
+ this.#offset = from + byteLength(complete);
125
+ const lines = split(complete);
126
+ // The first line is half a record whenever the read began mid-file, so it
127
+ // is dropped: what the fold reads are whole records or nothing.
128
+ if (from > 0) lines.shift();
129
+ this.deps.onSeed(lines);
130
+ }
131
+
132
+ async #read(): Promise<void> {
133
+ const size = await this.#stat();
134
+ if (size < this.#offset) {
135
+ // Shorter than what was already consumed: the file was replaced, so what
136
+ // was folded out of it describes nothing, and reading resumes from its
137
+ // beginning.
138
+ this.#offset = 0;
139
+ this.deps.onTruncated();
140
+ }
141
+ this.#size = size;
142
+ if (size === this.#offset) return;
143
+ const complete = whole(await this.#slice(this.#offset, size));
144
+ if (complete.length === 0) return;
145
+ const start = this.#offset;
146
+ const end = start + byteLength(complete);
147
+ this.#offset = end;
148
+ this.deps.onAppended({ lines: split(complete), start, end, size });
149
+ }
150
+
151
+ async #stat(): Promise<number> {
152
+ try {
153
+ return (await stat(this.path)).size;
154
+ } catch {
155
+ // Not there. Nothing was appended, and the poll keeps looking.
156
+ return this.#offset;
157
+ }
158
+ }
159
+
160
+ /** The bytes in a range, as text. A range that reads short — the file was
161
+ * truncated between the stat and the read — yields what was actually there. */
162
+ async #slice(from: number, to: number): Promise<string> {
163
+ const handle = await open(this.path, "r").catch(() => undefined);
164
+ if (handle === undefined) return "";
165
+ try {
166
+ const buffer = Buffer.alloc(to - from);
167
+ const { bytesRead } = await handle.read(buffer, 0, buffer.length, from);
168
+ return buffer.subarray(0, bytesRead).toString("utf8");
169
+ } finally {
170
+ await handle.close();
171
+ }
172
+ }
173
+ }
174
+
175
+ /** What of a read is whole records: everything up to and including the last
176
+ * newline. A transcript ends every record with one, so what follows the last
177
+ * is a record the writer has not finished. */
178
+ function whole(text: string): string {
179
+ const last = text.lastIndexOf("\n");
180
+ return last < 0 ? "" : text.slice(0, last + 1);
181
+ }
182
+
183
+ function split(complete: string): string[] {
184
+ return complete.split("\n").slice(0, -1);
185
+ }
186
+
187
+ function byteLength(text: string): number {
188
+ return Buffer.byteLength(text, "utf8");
189
+ }
190
+
191
+ /** How large the file is right now, or zero for one that is not there yet.
192
+ * Synchronous because the value is wanted before the first await. */
193
+ function sizeNow(path: string): number {
194
+ return statSync(path, { throwIfNoEntry: false })?.size ?? 0;
195
+ }
@@ -0,0 +1,162 @@
1
+ import type { InstanceId, Sid } from "@ccmsg/protocol";
2
+ import { topicParam, type TopicValue, type UpstreamResource } from "../topics/index.ts";
3
+ import { NO_FACTS, type TranscriptFacts, TranscriptFold } from "./fold.ts";
4
+ import { type Appended, TranscriptTail } from "./tail.ts";
5
+
6
+ export interface TranscriptsDeps {
7
+ readonly self: InstanceId;
8
+ /** Where a session's transcript is, as the session announced it (§5.1). A
9
+ * sid with no path is one that never said, and nothing is guessed for it. */
10
+ readonly pathOf: (sid: Sid) => string | undefined;
11
+ /** The one way a value reaches subscribers (§6.1). */
12
+ readonly publish: (topic: string, data: unknown) => void;
13
+ /** The fold now says something different about this session. What the fold
14
+ * settles is an input to the sessions domain (§5.1), so the domain that
15
+ * states those values is told to state them again. */
16
+ readonly onFacts: (sid: Sid) => void;
17
+ /** Overrides the confirmation poll, for a test that cannot wait. */
18
+ readonly pollMs?: number;
19
+ }
20
+
21
+ /** One tail and one fold per session, and the `transcript:<sid>` topic they
22
+ * feed (§3.3).
23
+ *
24
+ * The fold is one per session, not one per consumer: a line is read once and
25
+ * every value it settles is settled from that read, so the api error, the last
26
+ * human input and the appended bytes are three uses of one pass rather than
27
+ * three passes (M5).
28
+ *
29
+ * A tail runs while something wants it and stops when nothing does (§6.3).
30
+ * Subscription is one such want; a `hold` is the other, for the values the
31
+ * sessions domain states about a session nobody is watching the transcript of.
32
+ * They are counted together, so the last one to go is what stops the tail. */
33
+ export class Transcripts implements UpstreamResource {
34
+ readonly #followed = new Map<Sid, Followed>();
35
+
36
+ constructor(private readonly deps: TranscriptsDeps) {}
37
+
38
+ // --- UpstreamResource (§6.3)
39
+
40
+ start(topic: string): void {
41
+ const sid = topicParam(topic);
42
+ if (sid !== undefined) this.hold(sid);
43
+ }
44
+
45
+ stop(topic: string): void {
46
+ const sid = topicParam(topic);
47
+ if (sid !== undefined) this.release(sid);
48
+ }
49
+
50
+ /** Where the transcript ends as the subscription begins. What follows starts
51
+ * there, which is the whole of the snapshot for a topic whose frames are an
52
+ * append rather than a value (§6.2). A session whose transcript this
53
+ * instance cannot find has nothing to state, and the subscriber begins at
54
+ * the first thing appended after one appears. */
55
+ snapshot(topic: string): readonly TopicValue[] {
56
+ const sid = topicParam(topic);
57
+ const followed = sid === undefined ? undefined : this.#followed.get(sid);
58
+ if (sid === undefined || followed === undefined) return [];
59
+ return [{ instance: this.deps.self, data: { sid, size: followed.tail.size } }];
60
+ }
61
+
62
+ /** What the fold currently says about a session. Empty for one not being
63
+ * followed, which is the same as a transcript that has said nothing. */
64
+ facts(sid: Sid): TranscriptFacts {
65
+ return this.#followed.get(sid)?.fold.facts ?? NO_FACTS;
66
+ }
67
+
68
+ /** Whether a session's transcript is being followed, which is how "the
69
+ * subscription drives the resource" is observable from outside. */
70
+ following(sid: Sid): boolean {
71
+ return this.#followed.get(sid)?.tail.running === true;
72
+ }
73
+
74
+ /** Ask for a session's transcript to be followed. Each hold is released
75
+ * once; the tail runs until the last is. */
76
+ hold(sid: Sid): void {
77
+ const held = this.#followed.get(sid);
78
+ if (held !== undefined) {
79
+ held.holds += 1;
80
+ return;
81
+ }
82
+ const path = this.deps.pathOf(sid);
83
+ if (path === undefined) return;
84
+ const followed = this.#follow(sid, path);
85
+ this.#followed.set(sid, followed);
86
+ void followed.tail.start();
87
+ }
88
+
89
+ release(sid: Sid): void {
90
+ const held = this.#followed.get(sid);
91
+ if (held === undefined) return;
92
+ held.holds -= 1;
93
+ if (held.holds > 0) return;
94
+ this.#followed.delete(sid);
95
+ held.tail.stop();
96
+ // What the fold held goes with it: the values it derived describe a file
97
+ // this instance is no longer reading, and stating them from memory would
98
+ // outlive the reading that justified them.
99
+ this.deps.onFacts(sid);
100
+ }
101
+
102
+ /** Stop following everything. What shutdown reaches through the topics it
103
+ * drops; a hold taken outside a subscription needs the same door. */
104
+ stopAll(): void {
105
+ const followed = new Map(this.#followed);
106
+ this.#followed.clear();
107
+ for (const [sid, entry] of followed) {
108
+ entry.tail.stop();
109
+ this.deps.onFacts(sid);
110
+ }
111
+ }
112
+
113
+ #follow(sid: Sid, path: string): Followed {
114
+ const fold = new TranscriptFold();
115
+ const followed: Followed = {
116
+ holds: 1,
117
+ fold,
118
+ tail: new TranscriptTail(path, {
119
+ onSeed: (lines) => {
120
+ // The end of the file as it already stood: it settles what the fold
121
+ // says, and it is not an append, so nothing is published for it.
122
+ if (foldAll(fold, lines)) this.deps.onFacts(sid);
123
+ },
124
+ onAppended: (appended) => this.#appended(sid, fold, appended),
125
+ onTruncated: () => {
126
+ fold.reset();
127
+ this.deps.onFacts(sid);
128
+ },
129
+ ...(this.deps.pollMs === undefined ? {} : { pollMs: this.deps.pollMs }),
130
+ }),
131
+ };
132
+ return followed;
133
+ }
134
+
135
+ /** The one pass over what was appended: the lines go to the fold and to the
136
+ * topic, in that order, and are not read a second time for either (M5). */
137
+ #appended(sid: Sid, fold: TranscriptFold, appended: Appended): void {
138
+ const changed = foldAll(fold, appended.lines);
139
+ this.deps.publish(`transcript:${sid}`, {
140
+ sid,
141
+ lines: [...appended.lines],
142
+ start: appended.start,
143
+ end: appended.end,
144
+ size: appended.size,
145
+ });
146
+ if (changed) this.deps.onFacts(sid);
147
+ }
148
+ }
149
+
150
+ interface Followed {
151
+ holds: number;
152
+ readonly fold: TranscriptFold;
153
+ readonly tail: TranscriptTail;
154
+ }
155
+
156
+ function foldAll(fold: TranscriptFold, lines: readonly string[]): boolean {
157
+ let changed = false;
158
+ for (const line of lines) {
159
+ if (fold.line(line)) changed = true;
160
+ }
161
+ return changed;
162
+ }
@@ -0,0 +1,87 @@
1
+ import { accessSync, constants } from "node:fs";
2
+ import { isAbsolute } from "node:path";
3
+ import type { Capability } from "@ccmsg/protocol";
4
+ import { ConfigError, type UpstreamConfig } from "../instance/config.ts";
5
+
6
+ /** One exchange with the helper: a line in, a line out.
7
+ *
8
+ * The helper is a process that reads one JSON line and answers with one, and
9
+ * this is that process as everything above it needs to see it. Named rather
10
+ * than reached through directly so the wire can be exercised without a program
11
+ * on the host answering it. */
12
+ export interface HelperChannel {
13
+ write(line: string): Promise<void>;
14
+ /** The next line the helper wrote, or `undefined` once it has written its
15
+ * last — which is how a helper that died is told from one that is thinking. */
16
+ read(): Promise<string | undefined>;
17
+ kill(): void;
18
+ }
19
+
20
+ /** Read the helper the config names (§8.2).
21
+ *
22
+ * A helper that is named and cannot be run ends the start rather than leaving
23
+ * translation silently off (DV-Q9): an instance without the capability looks
24
+ * exactly like one nobody configured, and the operator who named a program
25
+ * meant to have it. */
26
+ export function translateSetup(config: UpstreamConfig, file: string): string | undefined {
27
+ const helper = config.translate_helper;
28
+ if (helper === undefined) return undefined;
29
+ if (!isAbsolute(helper)) {
30
+ throw new ConfigError(file, "upstream.translate_helper must be an absolute path");
31
+ }
32
+ try {
33
+ accessSync(helper, constants.X_OK);
34
+ } catch (cause) {
35
+ throw new ConfigError(
36
+ file,
37
+ `upstream.translate_helper ${helper} cannot be run (${String(cause)})`,
38
+ );
39
+ }
40
+ return helper;
41
+ }
42
+
43
+ /** Present exactly where a helper is configured: translation happens on this
44
+ * host or not at all, and a client is told which before it asks. */
45
+ export function translateCapabilities(helper?: string): Capability[] {
46
+ return helper === undefined ? [] : ["translate"];
47
+ }
48
+
49
+ /** Start the configured helper, speaking one JSON line at a time. */
50
+ export function spawnHelper(path: string): HelperChannel {
51
+ const child = Bun.spawn([path], { stdin: "pipe", stdout: "pipe", stderr: "inherit" });
52
+ const stdin = child.stdin;
53
+ const lines = readLines(child.stdout);
54
+ return {
55
+ write: async (line) => {
56
+ // The write is awaited because a full pipe answers with a promise rather
57
+ // than a count, and a line half-taken is a batch the helper never sees.
58
+ await stdin.write(line);
59
+ await stdin.flush();
60
+ },
61
+ read: () => lines(),
62
+ kill: () => {
63
+ child.kill();
64
+ },
65
+ };
66
+ }
67
+
68
+ /** The helper's output, one line at a time. */
69
+ function readLines(stream: ReadableStream<Uint8Array>): () => Promise<string | undefined> {
70
+ const reader = stream.getReader();
71
+ const decoder = new TextDecoder();
72
+ let buffer = "";
73
+ return async () => {
74
+ for (;;) {
75
+ const at = buffer.indexOf("\n");
76
+ if (at >= 0) {
77
+ const line = buffer.slice(0, at);
78
+ buffer = buffer.slice(at + 1);
79
+ if (line.trim() !== "") return line;
80
+ continue;
81
+ }
82
+ const { done, value } = await reader.read();
83
+ if (done) return undefined;
84
+ buffer += decoder.decode(value, { stream: true });
85
+ }
86
+ };
87
+ }
@@ -0,0 +1,2 @@
1
+ export * from "./helper.ts";
2
+ export * from "./translate.ts";
@@ -0,0 +1,127 @@
1
+ import type { TranslateResult, TranslateRunArgs, TranslateRunResult } from "@ccmsg/protocol";
2
+ import { type HandlerInput, OpError } from "../dispatch/index.ts";
3
+ import { type HelperChannel, spawnHelper } from "./helper.ts";
4
+
5
+ /** How long one batch may take before the helper is assumed wedged.
6
+ *
7
+ * The deadline grows with the input because the work does: a sentence comes
8
+ * back in seconds and a page takes minutes. The helper reports no progress, so
9
+ * a batch that outlives its deadline can only be ended by ending the helper —
10
+ * the next call starts a fresh one. */
11
+ const BASE_MS = 10_000;
12
+ const PER_100_CHARS_MS = 1_000;
13
+ const MAX_MS = 120_000;
14
+
15
+ export function deadlineMs(chars: number): number {
16
+ return Math.min(MAX_MS, BASE_MS + Math.ceil(chars / 100) * PER_100_CHARS_MS);
17
+ }
18
+
19
+ export interface TranslateDeps {
20
+ /** Starts the helper. Replaced in tests, which answer the same line protocol
21
+ * without a program on the host. */
22
+ readonly start?: () => HelperChannel;
23
+ readonly deadlineMs?: (chars: number) => number;
24
+ }
25
+
26
+ /** The host's translator, kept running between calls.
27
+ *
28
+ * Resident because starting it is the expensive part (DR-0023 §3.1): the
29
+ * process loads a translation session once and answers from it, so a batch per
30
+ * process would pay that cost on every keystroke's worth of text. One batch is
31
+ * in flight at a time — the helper answers one line per line it is given, and
32
+ * two batches sharing that channel could not tell the answers apart. */
33
+ export class Translate {
34
+ #helper: HelperChannel | undefined;
35
+ #queue: Promise<unknown> = Promise.resolve();
36
+ #next = 0;
37
+
38
+ constructor(
39
+ private readonly helperPath: string,
40
+ private readonly deps: TranslateDeps = {},
41
+ ) {}
42
+
43
+ async run(args: TranslateRunArgs): Promise<TranslateRunResult> {
44
+ // Nothing to translate needs no helper, and starting one to answer with an
45
+ // empty list would make the empty batch the probe it is not.
46
+ if (args.texts.length === 0) return { results: [] };
47
+ const done = this.#queue.then(
48
+ () => this.#exchange(args.texts),
49
+ () => this.#exchange(args.texts),
50
+ );
51
+ this.#queue = done.catch(() => undefined);
52
+ return { results: await done };
53
+ }
54
+
55
+ /** Stop the helper. What shutdown calls, and what a wedged batch does before
56
+ * it gives up. */
57
+ stop(): void {
58
+ this.#helper?.kill();
59
+ this.#helper = undefined;
60
+ }
61
+
62
+ async #exchange(texts: readonly string[]): Promise<TranslateResult[]> {
63
+ const helper = (this.#helper ??= (this.deps.start ?? (() => spawnHelper(this.helperPath)))());
64
+ const id = `${(this.#next += 1)}`;
65
+ const chars = texts.reduce((total, text) => total + text.length, 0);
66
+ const budget = (this.deps.deadlineMs ?? deadlineMs)(chars);
67
+ let expired: ReturnType<typeof setTimeout> | undefined;
68
+ try {
69
+ await helper.write(`${JSON.stringify({ id, texts })}\n`);
70
+ const line = await Promise.race([
71
+ helper.read(),
72
+ new Promise<never>((_resolve, reject) => {
73
+ expired = setTimeout(() => {
74
+ reject(new Error(`the helper did not answer within ${budget}ms`));
75
+ }, budget);
76
+ }),
77
+ ]);
78
+ if (line === undefined) throw new Error("the helper stopped before it answered");
79
+ return read(line, id, texts.length);
80
+ } catch (cause) {
81
+ // Whatever went wrong, this helper is no longer known to be in step with
82
+ // the line protocol, so it is ended and the next call starts a fresh one.
83
+ this.stop();
84
+ throw new OpError(
85
+ "translate_helper_failed",
86
+ `the translation helper failed: ${String(cause)}`,
87
+ );
88
+ } finally {
89
+ if (expired !== undefined) clearTimeout(expired);
90
+ }
91
+ }
92
+ }
93
+
94
+ /** One answer line, read as the batch it was meant to answer.
95
+ *
96
+ * The id and the count are both checked: an answer to another batch, or one
97
+ * holding a different number of results, would hand the caller texts that are
98
+ * not the ones it sent. */
99
+ function read(line: string, id: string, expected: number): TranslateResult[] {
100
+ const parsed: unknown = JSON.parse(line);
101
+ if (typeof parsed !== "object" || parsed === null) throw new Error("the answer is not an object");
102
+ const fields = parsed as Record<string, unknown>;
103
+ if (fields["id"] !== id) throw new Error(`the answer names batch ${String(fields["id"])}`);
104
+ const results = fields["results"];
105
+ if (!Array.isArray(results) || results.length !== expected) {
106
+ throw new Error(
107
+ `the answer holds ${Array.isArray(results) ? results.length : 0} of ${expected}`,
108
+ );
109
+ }
110
+ return results.map((entry): TranslateResult => {
111
+ const item = (typeof entry === "object" && entry !== null ? entry : {}) as Record<
112
+ string,
113
+ unknown
114
+ >;
115
+ const text = item["text"];
116
+ if (item["ok"] === true && typeof text === "string") return { ok: true, text };
117
+ const error = item["error"];
118
+ return { ok: false, error: typeof error === "string" ? error : "the helper stated no reason" };
119
+ });
120
+ }
121
+
122
+ export function translateHandlers(translate: Translate) {
123
+ return {
124
+ translate_run: (input: HandlerInput): Promise<TranslateRunResult> =>
125
+ translate.run(input.args as unknown as TranslateRunArgs),
126
+ };
127
+ }