@code-yeongyu/senpi-agent-core 2026.9.30 → 2026.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -20
- package/dist/agent-loop.d.ts +27 -3
- package/dist/agent-loop.js +192 -56
- package/dist/agent.d.ts +21 -7
- package/dist/agent.js +38 -18
- package/dist/harness/execution/tools.d.ts +1 -1
- package/dist/harness/execution/tools.js +0 -2
- package/dist/harness/messages.js +1 -0
- package/dist/harness/pico3/bash.d.ts +10 -0
- package/dist/harness/pico3/bash.js +25 -0
- package/dist/harness/pico3/bounded.d.ts +21 -0
- package/dist/harness/pico3/bounded.js +95 -0
- package/dist/harness/pico3/chord.d.ts +41 -0
- package/dist/harness/pico3/chord.js +147 -0
- package/dist/harness/pico3/context.d.ts +16 -0
- package/dist/harness/pico3/context.js +88 -0
- package/dist/harness/pico3/harness.d.ts +198 -0
- package/dist/harness/pico3/harness.js +645 -0
- package/dist/harness/pico3/hooks.d.ts +10 -0
- package/dist/harness/pico3/hooks.js +33 -0
- package/dist/harness/pico3/index.d.ts +19 -0
- package/dist/harness/pico3/index.js +15 -0
- package/dist/harness/pico3/jsonl.d.ts +58 -0
- package/dist/harness/pico3/jsonl.js +322 -0
- package/dist/harness/pico3/kinds/collapse.d.ts +49 -0
- package/dist/harness/pico3/kinds/collapse.js +191 -0
- package/dist/harness/pico3/kinds/entries.d.ts +78 -0
- package/dist/harness/pico3/kinds/entries.js +13 -0
- package/dist/harness/pico3/kinds/frames.d.ts +9 -0
- package/dist/harness/pico3/kinds/frames.js +75 -0
- package/dist/harness/pico3/kinds/generation.d.ts +94 -0
- package/dist/harness/pico3/kinds/generation.js +504 -0
- package/dist/harness/pico3/kinds/job.d.ts +44 -0
- package/dist/harness/pico3/kinds/job.js +143 -0
- package/dist/harness/pico3/kinds/plugin.d.ts +15 -0
- package/dist/harness/pico3/kinds/plugin.js +35 -0
- package/dist/harness/pico3/kinds/post-tools.d.ts +22 -0
- package/dist/harness/pico3/kinds/post-tools.js +145 -0
- package/dist/harness/pico3/kinds/task-api.d.ts +3 -0
- package/dist/harness/pico3/kinds/task-api.js +45 -0
- package/dist/harness/pico3/kinds/tool.d.ts +43 -0
- package/dist/harness/pico3/kinds/tool.js +373 -0
- package/dist/harness/pico3/legacy-tracker.d.ts +11 -0
- package/dist/harness/pico3/legacy-tracker.js +39 -0
- package/dist/harness/pico3/membrane.d.ts +23 -0
- package/dist/harness/pico3/membrane.js +140 -0
- package/dist/harness/pico3/memory.d.ts +53 -0
- package/dist/harness/pico3/memory.js +265 -0
- package/dist/harness/pico3/scheduler.d.ts +49 -0
- package/dist/harness/pico3/scheduler.js +437 -0
- package/dist/harness/pico3/session.d.ts +264 -0
- package/dist/harness/pico3/session.js +1328 -0
- package/dist/harness/pico3/system.d.ts +126 -0
- package/dist/harness/pico3/system.js +244 -0
- package/dist/harness/pico3/types.d.ts +973 -0
- package/dist/harness/pico3/types.js +94 -0
- package/dist/harness/pico3/view.d.ts +34 -0
- package/dist/harness/pico3/view.js +404 -0
- package/dist/harness/runtime/drive/tool-placement.js +2 -27
- package/dist/harness/telemetry.d.ts +36 -36
- package/dist/harness/tools/image.js +1 -1
- package/dist/proxy.d.ts +2 -2
- package/dist/types.d.ts +111 -34
- package/package.json +9 -5
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Experimental Pico3 kernel API.
|
|
3
|
+
*
|
|
4
|
+
* This subpath is intentionally separate from the package root while the kernel
|
|
5
|
+
* and its Chord integration are being validated.
|
|
6
|
+
*/
|
|
7
|
+
export { bashTool } from "./bash.ts";
|
|
8
|
+
export { Bounded } from "./bounded.ts";
|
|
9
|
+
export type { ChordViewBridge, ChordViewBridgeOptions, PublishedConversationView, } from "./chord.ts";
|
|
10
|
+
export { attachChordView, createPicoConversationService, PicoConversationService, PicoHarnessService, } from "./chord.ts";
|
|
11
|
+
export type { BuiltinKinds, ConfigFor, ConversationHandle, HarnessOptions, InputHandle, Watch, } from "./harness.ts";
|
|
12
|
+
export { applyEnvelope, captureActiveTranscript, entries, Harness, isCoreKind, kinds, WATCH_CAPACITY, withAbortSignal, } from "./harness.ts";
|
|
13
|
+
export { JsonlStorage } from "./jsonl.ts";
|
|
14
|
+
export { MemoryStorage } from "./memory.ts";
|
|
15
|
+
export type { SystemSection } from "./system.ts";
|
|
16
|
+
export { defineSystemSection, systemSections } from "./system.ts";
|
|
17
|
+
export type { AbortClosure, AnyKind, AnyToolDeclaration, AssistantMessage, BeforeToolApi, Checkpoint, CheckpointAt, Closure, Completion, ConfigFacade, ConfigOf, ConfigOfKinds, ConfigShape, ContextEdit, ContextView, Conversation, ConversationSpec, ConversationView, DeferredHandle, DisjointConfig, DocRef, Entry, EntryInput, EntryKind, EntryRef, EntryScan, Envelope, GenerationStatus, HookApi, HookBinding, HookInfo, HookResult, HookRunner, HooksOf, HostTx, Id, Input, InputOf, JsonObject, JsonPrimitive, JsonValue, Kind, KindConfig, KindTypes, Message, Model, ModelRef, Models, Namespace, NamespaceDefaults, NewEntry, Outcome, OwnedConversation, OwnedConversationSpec, PhaseHandler, PluginHandler, ProcessHost, ProcessSpec, ProcessStatus, QueuedInput, RequestMessage, RequestOptions, RetryPolicy, RewindableState, Runtime, SendInput, Seq, SessionState, SlotOf, Step, StickyState, Storage, Stored, StoredMessage, SystemMessage, Task, TaskOf, TaskRef, TaskScan, TaskTx, ThinkingLevel, ToolApi, ToolCall, ToolControl, ToolDeclaration, ToolDiagnostic, ToolResult, ToolResultMessage, ToolSlot, TurnState, TurnView, UserInput, UserMessage, ViewEvent, Write, } from "./types.ts";
|
|
18
|
+
export { Closed, CollapseInProgress, ConversationBusy, defineEntry, defineTask, Faulted, Forbidden, GenerationInProgress, memoOnce, ReadAfterWrite, TaskContractFault, toStored, } from "./types.ts";
|
|
19
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Experimental Pico3 kernel API.
|
|
3
|
+
*
|
|
4
|
+
* This subpath is intentionally separate from the package root while the kernel
|
|
5
|
+
* and its Chord integration are being validated.
|
|
6
|
+
*/
|
|
7
|
+
export { bashTool } from "./bash.js";
|
|
8
|
+
export { Bounded } from "./bounded.js";
|
|
9
|
+
export { attachChordView, createPicoConversationService, PicoConversationService, PicoHarnessService, } from "./chord.js";
|
|
10
|
+
export { applyEnvelope, captureActiveTranscript, entries, Harness, isCoreKind, kinds, WATCH_CAPACITY, withAbortSignal, } from "./harness.js";
|
|
11
|
+
export { JsonlStorage } from "./jsonl.js";
|
|
12
|
+
export { MemoryStorage } from "./memory.js";
|
|
13
|
+
export { defineSystemSection, systemSections } from "./system.js";
|
|
14
|
+
export { Closed, CollapseInProgress, ConversationBusy, defineEntry, defineTask, Faulted, Forbidden, GenerationInProgress, memoOnce, ReadAfterWrite, TaskContractFault, toStored, } from "./types.js";
|
|
15
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import type { Context } from "@earendil-works/chord";
|
|
2
|
+
import { MemoryStorage } from "./memory.ts";
|
|
3
|
+
import type { DocRef, Seq, Storage, Write } from "./types.ts";
|
|
4
|
+
/**
|
|
5
|
+
* Files in a session directory:
|
|
6
|
+
* main.jsonl conversations, entries, inputs, rewindable/session doc ops, each task's
|
|
7
|
+
* CREATE and TERMINAL records, and one marker per commit. Append-only, never rewritten.
|
|
8
|
+
* sticky-<conversation>.jsonl sticky doc ops; rewritten from its last base by `truncate` (on the Session line)
|
|
9
|
+
* task-<id>.jsonl a live task's intermediate patches (running, checkpoints); unlinked after its
|
|
10
|
+
* terminal record is published in main
|
|
11
|
+
*
|
|
12
|
+
* Publication (§10): one commit `Seq`; sidecar records are appended first, then exactly one main
|
|
13
|
+
* record, last, listing the sidecar refs it expects — the publication point. Replay applies a sidecar
|
|
14
|
+
* record only when main has the marker for that seq naming that file; unconfirmed sidecar tails are
|
|
15
|
+
* ignored. Every record carries the committed-ID high-water. Bytes after the last newline of any file
|
|
16
|
+
* are a torn write and are truncated before the file is opened for append. There is no compaction.
|
|
17
|
+
*
|
|
18
|
+
* With `fsync: false`, recovery covers process termination while the OS and filesystem remain alive.
|
|
19
|
+
* It does not promise that an acknowledged commit survives power loss, kernel/host failure,
|
|
20
|
+
* storage-cache loss, or a filesystem that loses/reorders completed writes. An acknowledged tail may
|
|
21
|
+
* roll back, or a surviving main marker may reference a missing sidecar and make open fail rather
|
|
22
|
+
* than expose partial state. A lost pre-effect checkpoint can cause an external effect to be attempted
|
|
23
|
+
* again; external idempotency remains required.
|
|
24
|
+
*
|
|
25
|
+
* With `fsync: true`, each sidecar is flushed before the main publication marker is flushed. This
|
|
26
|
+
* strengthens file-data durability and ordering across machine failure. It is not a complete database
|
|
27
|
+
* guarantee: newly created files, renames, and unlinks are not followed by a parent-directory fsync,
|
|
28
|
+
* and storage hardware/filesystems may provide weaker guarantees.
|
|
29
|
+
*
|
|
30
|
+
* One process owns a directory at a time; a second process is unsupported.
|
|
31
|
+
*/
|
|
32
|
+
export declare class JsonlStorage extends MemoryStorage implements Storage {
|
|
33
|
+
private mainFd;
|
|
34
|
+
private readonly sidecars;
|
|
35
|
+
private readonly taskSidecars;
|
|
36
|
+
private readonly dir;
|
|
37
|
+
readonly fsync: boolean;
|
|
38
|
+
private closedOnce;
|
|
39
|
+
private constructor();
|
|
40
|
+
static open(dir: string, opts?: {
|
|
41
|
+
fsync?: boolean;
|
|
42
|
+
}): Promise<JsonlStorage>;
|
|
43
|
+
/** For tests and tooling: sizes of every file in the directory. */
|
|
44
|
+
sizes(): {
|
|
45
|
+
[file: string]: number;
|
|
46
|
+
};
|
|
47
|
+
private replay;
|
|
48
|
+
commit(writes: readonly Write[], ctx: Context): Promise<Seq>;
|
|
49
|
+
private nextIdAfter;
|
|
50
|
+
private append;
|
|
51
|
+
private sidecar;
|
|
52
|
+
private taskSidecar;
|
|
53
|
+
private retireTaskSidecar;
|
|
54
|
+
/** Rewrite the sticky sidecar from its last base: temp file, optional fsync, rename. Called on the Session line. */
|
|
55
|
+
truncate(ref: DocRef, _ctx?: Context): Promise<void>;
|
|
56
|
+
close(_ctx?: Context): Promise<void>;
|
|
57
|
+
}
|
|
58
|
+
//# sourceMappingURL=jsonl.d.ts.map
|
|
@@ -0,0 +1,322 @@
|
|
|
1
|
+
import { closeSync, existsSync, fsyncSync, ftruncateSync, mkdirSync, openSync, readdirSync, readFileSync, renameSync, statSync, unlinkSync, writeSync, } from "node:fs";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context";
|
|
4
|
+
import { isBase } from "@earendil-works/chord/delta";
|
|
5
|
+
import { MemoryStorage } from "./memory.js";
|
|
6
|
+
/**
|
|
7
|
+
* Files in a session directory:
|
|
8
|
+
* main.jsonl conversations, entries, inputs, rewindable/session doc ops, each task's
|
|
9
|
+
* CREATE and TERMINAL records, and one marker per commit. Append-only, never rewritten.
|
|
10
|
+
* sticky-<conversation>.jsonl sticky doc ops; rewritten from its last base by `truncate` (on the Session line)
|
|
11
|
+
* task-<id>.jsonl a live task's intermediate patches (running, checkpoints); unlinked after its
|
|
12
|
+
* terminal record is published in main
|
|
13
|
+
*
|
|
14
|
+
* Publication (§10): one commit `Seq`; sidecar records are appended first, then exactly one main
|
|
15
|
+
* record, last, listing the sidecar refs it expects — the publication point. Replay applies a sidecar
|
|
16
|
+
* record only when main has the marker for that seq naming that file; unconfirmed sidecar tails are
|
|
17
|
+
* ignored. Every record carries the committed-ID high-water. Bytes after the last newline of any file
|
|
18
|
+
* are a torn write and are truncated before the file is opened for append. There is no compaction.
|
|
19
|
+
*
|
|
20
|
+
* With `fsync: false`, recovery covers process termination while the OS and filesystem remain alive.
|
|
21
|
+
* It does not promise that an acknowledged commit survives power loss, kernel/host failure,
|
|
22
|
+
* storage-cache loss, or a filesystem that loses/reorders completed writes. An acknowledged tail may
|
|
23
|
+
* roll back, or a surviving main marker may reference a missing sidecar and make open fail rather
|
|
24
|
+
* than expose partial state. A lost pre-effect checkpoint can cause an external effect to be attempted
|
|
25
|
+
* again; external idempotency remains required.
|
|
26
|
+
*
|
|
27
|
+
* With `fsync: true`, each sidecar is flushed before the main publication marker is flushed. This
|
|
28
|
+
* strengthens file-data durability and ordering across machine failure. It is not a complete database
|
|
29
|
+
* guarantee: newly created files, renames, and unlinks are not followed by a parent-directory fsync,
|
|
30
|
+
* and storage hardware/filesystems may provide weaker guarantees.
|
|
31
|
+
*
|
|
32
|
+
* One process owns a directory at a time; a second process is unsupported.
|
|
33
|
+
*/
|
|
34
|
+
export class JsonlStorage extends MemoryStorage {
|
|
35
|
+
constructor(dir, fsync) {
|
|
36
|
+
super();
|
|
37
|
+
this.sidecars = new Map(); // sticky docs, by conversation
|
|
38
|
+
this.taskSidecars = new Map();
|
|
39
|
+
this.closedOnce = false;
|
|
40
|
+
this.dir = dir;
|
|
41
|
+
this.fsync = fsync;
|
|
42
|
+
this.mainFd = openSync(join(dir, "main.jsonl"), "a");
|
|
43
|
+
}
|
|
44
|
+
static async open(dir, opts = {}) {
|
|
45
|
+
mkdirSync(dir, { recursive: true });
|
|
46
|
+
for (const f of readdirSync(dir))
|
|
47
|
+
if (f.endsWith(".jsonl"))
|
|
48
|
+
truncateTornTail(join(dir, f));
|
|
49
|
+
const storage = new JsonlStorage(dir, opts.fsync ?? true);
|
|
50
|
+
try {
|
|
51
|
+
await storage.replay();
|
|
52
|
+
return storage;
|
|
53
|
+
}
|
|
54
|
+
catch (error) {
|
|
55
|
+
await storage.close();
|
|
56
|
+
throw error;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
/** For tests and tooling: sizes of every file in the directory. */
|
|
60
|
+
sizes() {
|
|
61
|
+
const out = {};
|
|
62
|
+
for (const f of readdirSync(this.dir))
|
|
63
|
+
out[f] = statSync(join(this.dir, f)).size;
|
|
64
|
+
return out;
|
|
65
|
+
}
|
|
66
|
+
async replay() {
|
|
67
|
+
const read = (file) => {
|
|
68
|
+
if (!existsSync(file))
|
|
69
|
+
return [];
|
|
70
|
+
const bytes = readFileSync(file);
|
|
71
|
+
const out = [];
|
|
72
|
+
let start = 0;
|
|
73
|
+
let last = 0;
|
|
74
|
+
for (let end = bytes.indexOf(0x0a, start); end >= 0; end = bytes.indexOf(0x0a, start)) {
|
|
75
|
+
const line = bytes.subarray(start, end).toString("utf8");
|
|
76
|
+
start = end + 1;
|
|
77
|
+
if (line === "")
|
|
78
|
+
continue;
|
|
79
|
+
let record;
|
|
80
|
+
try {
|
|
81
|
+
record = JSON.parse(line);
|
|
82
|
+
}
|
|
83
|
+
catch (error) {
|
|
84
|
+
throw new Error(`${file}: malformed record: ${String(error)}`);
|
|
85
|
+
}
|
|
86
|
+
if (typeof record.seq !== "number" || typeof record.maxId !== "number" || !Array.isArray(record.writes))
|
|
87
|
+
throw new Error(`${file}: record lacks seq/maxId/writes`);
|
|
88
|
+
if (record.seq <= last)
|
|
89
|
+
throw new Error(`${file}: sequence not increasing at ${record.seq}`);
|
|
90
|
+
last = record.seq;
|
|
91
|
+
out.push({ record, end: start });
|
|
92
|
+
}
|
|
93
|
+
return out;
|
|
94
|
+
};
|
|
95
|
+
let maxId = 0;
|
|
96
|
+
// Main is the commit log. Confirmed = (seq → refs) for every marker.
|
|
97
|
+
const main = read(join(this.dir, "main.jsonl")).map(({ record }) => record);
|
|
98
|
+
const confirmed = new Map();
|
|
99
|
+
for (const record of main)
|
|
100
|
+
confirmed.set(record.seq, new Set(record.refs ?? []));
|
|
101
|
+
// Merge every file's records by seq so batches apply in commit order.
|
|
102
|
+
const sidecarFiles = readdirSync(this.dir).filter((file) => file.startsWith("sticky-") || file.startsWith("task-"));
|
|
103
|
+
const bySeq = new Map();
|
|
104
|
+
const found = new Map();
|
|
105
|
+
const retainedStickyBase = new Map();
|
|
106
|
+
const retiredTasks = new Map();
|
|
107
|
+
for (const record of main) {
|
|
108
|
+
for (const write of record.writes) {
|
|
109
|
+
if (write.type === "task.patch" && write.patch.status === "terminal")
|
|
110
|
+
retiredTasks.set(write.patch.id, record.seq);
|
|
111
|
+
}
|
|
112
|
+
bySeq.set(record.seq, [...record.writes]);
|
|
113
|
+
maxId = Math.max(maxId, record.maxId);
|
|
114
|
+
}
|
|
115
|
+
for (const file of sidecarFiles) {
|
|
116
|
+
const path = join(this.dir, file);
|
|
117
|
+
const records = read(path);
|
|
118
|
+
const first = records[0]?.record;
|
|
119
|
+
if (file.startsWith("sticky-") &&
|
|
120
|
+
first !== undefined &&
|
|
121
|
+
isBase(first.writes.flatMap((write) => (write.type === "doc" ? write.ops : [])))) {
|
|
122
|
+
retainedStickyBase.set(file, first.seq);
|
|
123
|
+
}
|
|
124
|
+
let confirmedEnd = 0;
|
|
125
|
+
let sawUnconfirmed = false;
|
|
126
|
+
for (const { record, end } of records) {
|
|
127
|
+
if (!confirmed.get(record.seq)?.has(file)) {
|
|
128
|
+
sawUnconfirmed = true;
|
|
129
|
+
continue;
|
|
130
|
+
}
|
|
131
|
+
if (sawUnconfirmed)
|
|
132
|
+
throw new Error(`${path}: confirmed record follows an unconfirmed tail at ${record.seq}`);
|
|
133
|
+
confirmedEnd = end;
|
|
134
|
+
const files = found.get(record.seq) ?? new Set();
|
|
135
|
+
files.add(file);
|
|
136
|
+
found.set(record.seq, files);
|
|
137
|
+
bySeq.get(record.seq).push(...record.writes);
|
|
138
|
+
maxId = Math.max(maxId, record.maxId);
|
|
139
|
+
}
|
|
140
|
+
if (sawUnconfirmed)
|
|
141
|
+
truncateTo(path, confirmedEnd);
|
|
142
|
+
}
|
|
143
|
+
for (const record of main) {
|
|
144
|
+
for (const file of record.refs ?? []) {
|
|
145
|
+
if (found.get(record.seq)?.has(file))
|
|
146
|
+
continue;
|
|
147
|
+
const retainedFrom = retainedStickyBase.get(file);
|
|
148
|
+
if (retainedFrom !== undefined && record.seq < retainedFrom)
|
|
149
|
+
continue;
|
|
150
|
+
if (file.startsWith("task-")) {
|
|
151
|
+
const id = Number(file.slice(5, -6));
|
|
152
|
+
const retiredAt = retiredTasks.get(id);
|
|
153
|
+
if (retiredAt !== undefined && record.seq < retiredAt)
|
|
154
|
+
continue;
|
|
155
|
+
}
|
|
156
|
+
throw new Error(`${join(this.dir, file)}: missing record for published sequence ${record.seq}`);
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
for (const seq of [...bySeq.keys()].sort((a, b) => a - b)) {
|
|
160
|
+
this.seq = seq - 1;
|
|
161
|
+
await super.commit(bySeq.get(seq), BACKGROUND_CONTEXT);
|
|
162
|
+
}
|
|
163
|
+
this.setNextId(maxId + 1);
|
|
164
|
+
// Task sidecars belong to live tasks only; a leftover for a terminal (or unknown) task cannot resurrect anything.
|
|
165
|
+
for (const f of sidecarFiles) {
|
|
166
|
+
if (!f.startsWith("task-"))
|
|
167
|
+
continue;
|
|
168
|
+
const id = Number(f.slice(5, -6));
|
|
169
|
+
const t = await super.task(id);
|
|
170
|
+
if (t === undefined || t.status === "terminal")
|
|
171
|
+
unlinkSync(join(this.dir, f));
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
async commit(writes, ctx) {
|
|
175
|
+
if (this.closedOnce)
|
|
176
|
+
throw new Error("storage closed");
|
|
177
|
+
const seq = this.seq + 1;
|
|
178
|
+
const apply = this.stage(writes, seq); // validates the whole batch first; throws before any file is touched
|
|
179
|
+
const maxId = this.nextIdAfter(writes);
|
|
180
|
+
const main = [];
|
|
181
|
+
const sticky = new Map();
|
|
182
|
+
const tasks = new Map();
|
|
183
|
+
const retired = [];
|
|
184
|
+
for (const w of writes) {
|
|
185
|
+
if (w.type === "doc" && w.ref.doc === "sticky")
|
|
186
|
+
(sticky.get(w.ref.conversationId) ?? sticky.set(w.ref.conversationId, []).get(w.ref.conversationId)).push(w);
|
|
187
|
+
else if (w.type === "task.patch" && w.patch.status !== "terminal")
|
|
188
|
+
(tasks.get(w.patch.id) ?? tasks.set(w.patch.id, []).get(w.patch.id)).push(w);
|
|
189
|
+
else {
|
|
190
|
+
main.push(w);
|
|
191
|
+
if (w.type === "task.patch" && w.patch.status === "terminal")
|
|
192
|
+
retired.push(w.patch.id);
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
const refs = [];
|
|
196
|
+
for (const [id, ws] of tasks) {
|
|
197
|
+
this.append(this.taskSidecar(id), { seq, maxId, writes: ws });
|
|
198
|
+
refs.push(`task-${id}.jsonl`);
|
|
199
|
+
}
|
|
200
|
+
for (const [c, ws] of sticky) {
|
|
201
|
+
this.append(this.sidecar(c), { seq, maxId, writes: ws });
|
|
202
|
+
refs.push(`sticky-${c}.jsonl`);
|
|
203
|
+
}
|
|
204
|
+
this.append(this.mainFd, { seq, maxId, writes: main, ...(refs.length ? { refs } : {}) }); // the publication point
|
|
205
|
+
apply();
|
|
206
|
+
this.seq = seq;
|
|
207
|
+
for (const id of retired)
|
|
208
|
+
this.retireTaskSidecar(id); // after the terminal record is durably published
|
|
209
|
+
void ctx;
|
|
210
|
+
return seq;
|
|
211
|
+
}
|
|
212
|
+
nextIdAfter(writes) {
|
|
213
|
+
let max = this.nextIdValue - 1;
|
|
214
|
+
for (const w of writes) {
|
|
215
|
+
const id = w.type === "conversation"
|
|
216
|
+
? w.conversation.id
|
|
217
|
+
: w.type === "entry"
|
|
218
|
+
? w.entry.id
|
|
219
|
+
: w.type === "task"
|
|
220
|
+
? w.task.id
|
|
221
|
+
: w.type === "input"
|
|
222
|
+
? w.input.id
|
|
223
|
+
: 0;
|
|
224
|
+
max = Math.max(max, id);
|
|
225
|
+
}
|
|
226
|
+
return max;
|
|
227
|
+
}
|
|
228
|
+
append(fd, record) {
|
|
229
|
+
writeSync(fd, `${JSON.stringify(record)}\n`);
|
|
230
|
+
if (this.fsync)
|
|
231
|
+
fsyncSync(fd);
|
|
232
|
+
}
|
|
233
|
+
sidecar(conversationId) {
|
|
234
|
+
let fd = this.sidecars.get(conversationId);
|
|
235
|
+
if (fd === undefined) {
|
|
236
|
+
fd = openSync(join(this.dir, `sticky-${conversationId}.jsonl`), "a");
|
|
237
|
+
this.sidecars.set(conversationId, fd);
|
|
238
|
+
}
|
|
239
|
+
return fd;
|
|
240
|
+
}
|
|
241
|
+
taskSidecar(id) {
|
|
242
|
+
let fd = this.taskSidecars.get(id);
|
|
243
|
+
if (fd === undefined) {
|
|
244
|
+
fd = openSync(join(this.dir, `task-${id}.jsonl`), "a");
|
|
245
|
+
this.taskSidecars.set(id, fd);
|
|
246
|
+
}
|
|
247
|
+
return fd;
|
|
248
|
+
}
|
|
249
|
+
retireTaskSidecar(id) {
|
|
250
|
+
const fd = this.taskSidecars.get(id);
|
|
251
|
+
if (fd !== undefined)
|
|
252
|
+
closeSync(fd);
|
|
253
|
+
this.taskSidecars.delete(id);
|
|
254
|
+
try {
|
|
255
|
+
unlinkSync(join(this.dir, `task-${id}.jsonl`));
|
|
256
|
+
}
|
|
257
|
+
catch { }
|
|
258
|
+
}
|
|
259
|
+
/** Rewrite the sticky sidecar from its last base: temp file, optional fsync, rename. Called on the Session line. */
|
|
260
|
+
async truncate(ref, _ctx) {
|
|
261
|
+
await super.truncate(ref);
|
|
262
|
+
if (ref.doc !== "sticky")
|
|
263
|
+
return;
|
|
264
|
+
const file = join(this.dir, `sticky-${ref.conversationId}.jsonl`);
|
|
265
|
+
if (!existsSync(file))
|
|
266
|
+
return;
|
|
267
|
+
const lines = readFileSync(file, "utf8").split("\n").filter(Boolean);
|
|
268
|
+
let start = 0;
|
|
269
|
+
for (let i = lines.length - 1; i >= 0; i--)
|
|
270
|
+
if (isBase(JSON.parse(lines[i]).writes.flatMap((w) => (w.type === "doc" ? w.ops : [])))) {
|
|
271
|
+
start = i;
|
|
272
|
+
break;
|
|
273
|
+
}
|
|
274
|
+
if (start === 0)
|
|
275
|
+
return;
|
|
276
|
+
const tmp = `${file}.tmp`;
|
|
277
|
+
const fd = openSync(tmp, "w");
|
|
278
|
+
writeSync(fd, lines
|
|
279
|
+
.slice(start)
|
|
280
|
+
.map((l) => `${l}\n`)
|
|
281
|
+
.join(""));
|
|
282
|
+
if (this.fsync)
|
|
283
|
+
fsyncSync(fd);
|
|
284
|
+
closeSync(fd);
|
|
285
|
+
const old = this.sidecars.get(ref.conversationId);
|
|
286
|
+
if (old !== undefined)
|
|
287
|
+
closeSync(old);
|
|
288
|
+
renameSync(tmp, file);
|
|
289
|
+
this.sidecars.set(ref.conversationId, openSync(file, "a"));
|
|
290
|
+
}
|
|
291
|
+
async close(_ctx) {
|
|
292
|
+
if (this.closedOnce)
|
|
293
|
+
return;
|
|
294
|
+
this.closedOnce = true;
|
|
295
|
+
await super.close();
|
|
296
|
+
closeSync(this.mainFd);
|
|
297
|
+
for (const fd of this.sidecars.values())
|
|
298
|
+
closeSync(fd);
|
|
299
|
+
for (const fd of this.taskSidecars.values())
|
|
300
|
+
closeSync(fd);
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
function truncateTo(file, length) {
|
|
304
|
+
const fd = openSync(file, "r+");
|
|
305
|
+
try {
|
|
306
|
+
ftruncateSync(fd, length);
|
|
307
|
+
}
|
|
308
|
+
finally {
|
|
309
|
+
closeSync(fd);
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
/** Bytes after the last newline are a torn write: cut them so a later append cannot extend a corrupt line. */
|
|
313
|
+
function truncateTornTail(file) {
|
|
314
|
+
const size = statSync(file).size;
|
|
315
|
+
if (size === 0)
|
|
316
|
+
return;
|
|
317
|
+
const buf = readFileSync(file);
|
|
318
|
+
const end = buf.lastIndexOf(0x0a);
|
|
319
|
+
if (end + 1 !== buf.length)
|
|
320
|
+
truncateTo(file, end + 1);
|
|
321
|
+
}
|
|
322
|
+
//# sourceMappingURL=jsonl.js.map
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import type { Context } from "@earendil-works/chord";
|
|
2
|
+
import { type CoreKind, type Entry, type HookInfo, type HookResult, type Id, type ModelRef, type RetryPolicy, type ThinkingLevel } from "../types.ts";
|
|
3
|
+
export type CollapseInput = {
|
|
4
|
+
reason: "threshold" | "manual" | "overflow";
|
|
5
|
+
through: Id;
|
|
6
|
+
instructions?: string;
|
|
7
|
+
};
|
|
8
|
+
type Base = {
|
|
9
|
+
expectedHead: Id | null;
|
|
10
|
+
instructions?: string;
|
|
11
|
+
model: ModelRef;
|
|
12
|
+
thinkingLevel: ThinkingLevel;
|
|
13
|
+
retry: RetryPolicy;
|
|
14
|
+
attempt: number;
|
|
15
|
+
};
|
|
16
|
+
export type CollapseCheckpoint = ({
|
|
17
|
+
phase: "summarizing";
|
|
18
|
+
} & Base) | ({
|
|
19
|
+
phase: "retrying";
|
|
20
|
+
untilMs: number;
|
|
21
|
+
lastError: string;
|
|
22
|
+
} & Base) | ({
|
|
23
|
+
phase: "prepared";
|
|
24
|
+
summary: string;
|
|
25
|
+
} & Base);
|
|
26
|
+
export type CollapseFailure = {
|
|
27
|
+
reason: "stale" | "declined" | "provider" | "retries_exhausted" | "no_model";
|
|
28
|
+
detail: string;
|
|
29
|
+
};
|
|
30
|
+
export interface CollapseHooks {
|
|
31
|
+
beforeCollapse(reason: "threshold" | "manual" | "overflow", through: Id, entries: Entry[], info: HookInfo, ctx: Context): HookResult<{
|
|
32
|
+
decline: true;
|
|
33
|
+
} | {
|
|
34
|
+
instructions?: string;
|
|
35
|
+
summary?: string;
|
|
36
|
+
}>;
|
|
37
|
+
}
|
|
38
|
+
export declare const collapseConfig: {
|
|
39
|
+
readonly rewindable: {
|
|
40
|
+
readonly threshold: number;
|
|
41
|
+
readonly keepRecent: number;
|
|
42
|
+
};
|
|
43
|
+
};
|
|
44
|
+
export declare const collapse: CoreKind<CollapseInput, CollapseCheckpoint, {
|
|
45
|
+
summary: Id;
|
|
46
|
+
}, CollapseFailure, null, CollapseHooks, typeof collapseConfig>;
|
|
47
|
+
export declare function chooseThrough(entries: Entry[], keepRecent: number): Id | undefined;
|
|
48
|
+
export {};
|
|
49
|
+
//# sourceMappingURL=collapse.d.ts.map
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
import { estimateContextTokens } from "@earendil-works/pi-ai/utils/estimate";
|
|
2
|
+
import { effectiveTools } from "../system.js";
|
|
3
|
+
import { toStored, } from "../types.js";
|
|
4
|
+
import { retryDecision } from "./generation.js";
|
|
5
|
+
const estimate = (messages) => estimateContextTokens(messages.filter((message) => message.role !== "system")).tokens;
|
|
6
|
+
export const collapseConfig = { rewindable: { threshold: 0, keepRecent: 20_000 } };
|
|
7
|
+
const failed = (reason, detail) => ({
|
|
8
|
+
status: "failed",
|
|
9
|
+
failure: { reason, detail },
|
|
10
|
+
});
|
|
11
|
+
const baseOf = (checkpoint) => {
|
|
12
|
+
const { phase: _phase, ...rest } = checkpoint;
|
|
13
|
+
const { untilMs: _untilMs, lastError: _lastError, summary: _summary, ...base } = rest;
|
|
14
|
+
return base;
|
|
15
|
+
};
|
|
16
|
+
export const collapse = {
|
|
17
|
+
name: "pi.collapse",
|
|
18
|
+
config: collapseConfig,
|
|
19
|
+
inflight: ["summarizing"],
|
|
20
|
+
async initial(task, runtime, ctx) {
|
|
21
|
+
const state = await runtime.rewindable(task.conversationId, ctx);
|
|
22
|
+
if (state.model === undefined)
|
|
23
|
+
return { done: () => failed("no_model", "no model configured") };
|
|
24
|
+
const sticky = await runtime.sticky(task.conversationId, ctx);
|
|
25
|
+
const head = await runtime.newestEntry(task.conversationId, { withHead: true }, ctx);
|
|
26
|
+
const { entries } = await runtime.context(task.conversationId, task.input.through, ctx);
|
|
27
|
+
let decision = {};
|
|
28
|
+
await runtime.hooks.each(ctx, (hook, api) => hook.beforeCollapse?.(task.input.reason, task.input.through, entries, api, ctx), (value) => {
|
|
29
|
+
if (value === undefined)
|
|
30
|
+
return;
|
|
31
|
+
decision = value;
|
|
32
|
+
return true;
|
|
33
|
+
});
|
|
34
|
+
if ("decline" in decision)
|
|
35
|
+
return { done: () => failed("declined", "declined by beforeCollapse") };
|
|
36
|
+
const choice = decision;
|
|
37
|
+
const base = {
|
|
38
|
+
expectedHead: head?.id ?? null,
|
|
39
|
+
...((choice.instructions ?? task.input.instructions)
|
|
40
|
+
? { instructions: choice.instructions ?? task.input.instructions }
|
|
41
|
+
: {}),
|
|
42
|
+
model: state.model,
|
|
43
|
+
thinkingLevel: state.thinkingLevel,
|
|
44
|
+
retry: sticky.retry,
|
|
45
|
+
attempt: 1,
|
|
46
|
+
};
|
|
47
|
+
if (choice.summary !== undefined) {
|
|
48
|
+
return {
|
|
49
|
+
next: async (tx, current) => (await headMoved(tx, current.conversationId, base))
|
|
50
|
+
? failed("stale", "head moved during beforeCollapse")
|
|
51
|
+
: { phase: "prepared", summary: choice.summary, ...base },
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
return summarizeNow(task, base, runtime, ctx, "check head");
|
|
55
|
+
},
|
|
56
|
+
phases: {
|
|
57
|
+
async summarizing(task, runtime) {
|
|
58
|
+
return afterFailure(baseOf(task.checkpoint), null, "interrupted", runtime);
|
|
59
|
+
},
|
|
60
|
+
async retrying(task, runtime, ctx) {
|
|
61
|
+
await runtime.sleep(task.checkpoint.untilMs, ctx);
|
|
62
|
+
return summarizeNow(task, { ...baseOf(task.checkpoint), attempt: task.checkpoint.attempt + 1 }, runtime, ctx);
|
|
63
|
+
},
|
|
64
|
+
async prepared(task, runtime) {
|
|
65
|
+
const { summary } = task.checkpoint;
|
|
66
|
+
const base = baseOf(task.checkpoint);
|
|
67
|
+
return {
|
|
68
|
+
done: async (tx, current) => {
|
|
69
|
+
const { head, entries } = await tx.context(current.conversationId);
|
|
70
|
+
if ((head?.id ?? null) !== base.expectedHead)
|
|
71
|
+
return failed("stale", "head moved during collapse");
|
|
72
|
+
const retained = entries.find((entry) => entry.id > task.input.through);
|
|
73
|
+
const id = tx.appendEntry(current.conversationId, {
|
|
74
|
+
kind: "pi.summary",
|
|
75
|
+
data: { through: task.input.through },
|
|
76
|
+
model: [{ role: "user", content: summary, timestamp: runtime.now() }],
|
|
77
|
+
head: retained?.id ?? "self",
|
|
78
|
+
});
|
|
79
|
+
return { status: "completed", result: { summary: id } };
|
|
80
|
+
},
|
|
81
|
+
};
|
|
82
|
+
},
|
|
83
|
+
},
|
|
84
|
+
async abort() {
|
|
85
|
+
return async () => null;
|
|
86
|
+
},
|
|
87
|
+
};
|
|
88
|
+
const headMoved = async (tx, conversationId, base) => ((await tx.newestEntry(conversationId, { withHead: true }))?.id ?? null) !== base.expectedHead;
|
|
89
|
+
async function summarizeNow(task, base, runtime, ctx, check) {
|
|
90
|
+
const stale = await runtime.commit(async (tx, current) => {
|
|
91
|
+
if (check && (await headMoved(tx, current.conversationId, base)))
|
|
92
|
+
return true;
|
|
93
|
+
tx.checkpoint({ phase: "summarizing", ...base });
|
|
94
|
+
return false;
|
|
95
|
+
}, ctx);
|
|
96
|
+
if (stale)
|
|
97
|
+
return { done: () => failed("stale", "head moved during beforeCollapse") };
|
|
98
|
+
const model = runtime.models.resolve(base.model);
|
|
99
|
+
if (model === undefined)
|
|
100
|
+
return { done: () => failed("no_model", "model unavailable") };
|
|
101
|
+
const { messages } = await runtime.context(task.conversationId, task.input.through, ctx);
|
|
102
|
+
const tools = effectiveTools(messages);
|
|
103
|
+
const request = [
|
|
104
|
+
...messages,
|
|
105
|
+
...(tools.length > 0
|
|
106
|
+
? [toStored({ role: "system", content: "", toolsRemoved: tools, timestamp: runtime.now() })]
|
|
107
|
+
: []),
|
|
108
|
+
{
|
|
109
|
+
role: "user",
|
|
110
|
+
content: `${base.instructions ?? "Summarize the conversation so far for continuation."}\n\nRespond with the summary only.`,
|
|
111
|
+
timestamp: runtime.now(),
|
|
112
|
+
},
|
|
113
|
+
];
|
|
114
|
+
let message;
|
|
115
|
+
try {
|
|
116
|
+
for await (const event of runtime.models.stream(model, { messages: request, thinkingLevel: base.thinkingLevel }, ctx)) {
|
|
117
|
+
if (event.type === "done") {
|
|
118
|
+
message = event.message;
|
|
119
|
+
break;
|
|
120
|
+
}
|
|
121
|
+
if (event.type === "error") {
|
|
122
|
+
message = event.error;
|
|
123
|
+
break;
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
catch (error) {
|
|
128
|
+
if (ctx.abortSignal?.aborted)
|
|
129
|
+
throw error;
|
|
130
|
+
return afterFailure(base, null, String(error), runtime);
|
|
131
|
+
}
|
|
132
|
+
if (message === undefined)
|
|
133
|
+
return afterFailure(base, null, "no message", runtime);
|
|
134
|
+
if (message.content.some((content) => content.type === "toolCall")) {
|
|
135
|
+
return afterFailure(base, { ...message, stopReason: "error" }, "summarizer returned tool calls", runtime);
|
|
136
|
+
}
|
|
137
|
+
if (message.stopReason === "error")
|
|
138
|
+
return afterFailure(base, message, message.errorMessage ?? "provider error", runtime);
|
|
139
|
+
const summary = message.content
|
|
140
|
+
.filter((content) => content.type === "text")
|
|
141
|
+
.map((content) => content.text)
|
|
142
|
+
.join("");
|
|
143
|
+
return { next: { phase: "prepared", summary, ...base } };
|
|
144
|
+
}
|
|
145
|
+
function afterFailure(base, message, detail, runtime) {
|
|
146
|
+
const decision = retryDecision(base, message, runtime.now());
|
|
147
|
+
if (decision.kind === "fail")
|
|
148
|
+
return { done: () => failed(decision.reason, detail) };
|
|
149
|
+
return {
|
|
150
|
+
next: (tx, current) => {
|
|
151
|
+
tx.emit({
|
|
152
|
+
type: "compaction.retrying",
|
|
153
|
+
taskId: current.id,
|
|
154
|
+
attempt: base.attempt,
|
|
155
|
+
retryAt: decision.untilMs,
|
|
156
|
+
error: detail,
|
|
157
|
+
});
|
|
158
|
+
return { phase: "retrying", ...base, untilMs: decision.untilMs, lastError: detail };
|
|
159
|
+
},
|
|
160
|
+
};
|
|
161
|
+
}
|
|
162
|
+
export function chooseThrough(entries, keepRecent) {
|
|
163
|
+
const exchanges = [];
|
|
164
|
+
let open;
|
|
165
|
+
for (const entry of entries) {
|
|
166
|
+
const role = entry.model?.[0]?.role;
|
|
167
|
+
const tokens = estimate((entry.model ?? []));
|
|
168
|
+
if (role === "assistant") {
|
|
169
|
+
open = { last: entry.id, tokens };
|
|
170
|
+
exchanges.push(open);
|
|
171
|
+
}
|
|
172
|
+
else if (role === "toolResult" && open) {
|
|
173
|
+
open.last = entry.id;
|
|
174
|
+
open.tokens += tokens;
|
|
175
|
+
}
|
|
176
|
+
else {
|
|
177
|
+
open = undefined;
|
|
178
|
+
exchanges.push({ last: entry.id, tokens });
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
let retained = 0;
|
|
182
|
+
let index = exchanges.length - 1;
|
|
183
|
+
while (index >= 0 && retained + exchanges[index].tokens <= keepRecent)
|
|
184
|
+
retained += exchanges[index--].tokens;
|
|
185
|
+
if (index < 0)
|
|
186
|
+
return undefined;
|
|
187
|
+
if (index === exchanges.length - 1)
|
|
188
|
+
return exchanges[index - 1]?.last;
|
|
189
|
+
return exchanges[index].last;
|
|
190
|
+
}
|
|
191
|
+
//# sourceMappingURL=collapse.js.map
|