@ixo/oracle-runtime-workers 0.37.2 → 0.39.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +4 -4
- package/src/attachments/channel-links.ts +74 -7
- package/src/attachments/channel.ts +15 -4
- package/src/attachments/content-blocks.ts +19 -16
- package/src/attachments/download.ts +173 -50
- package/src/attachments/index.ts +6 -5
- package/src/attachments/pipeline.ts +117 -49
- package/src/attachments/resize.ts +142 -0
- package/src/attachments/retention.ts +26 -70
- package/src/attachments/room-media.ts +643 -0
- package/src/attachments/route.ts +47 -4
- package/src/attachments/sandbox-archive.ts +15 -4
- package/src/attachments/sanitize.ts +24 -13
- package/src/attachments/view.ts +82 -38
- package/src/channels/replies.ts +8 -4
- package/src/core/capability-expiry.ts +62 -0
- package/src/core/capability-router.ts +126 -95
- package/src/core/context-budget.ts +116 -4
- package/src/core/domain-context/document.ts +74 -2
- package/src/core/domain-context/index.ts +1 -0
- package/src/core/domain-context/resolver.ts +9 -6
- package/src/core/domain-context/turn-context.ts +62 -13
- package/src/core/domain-context/types.ts +17 -0
- package/src/core/env.ts +21 -2
- package/src/core/index.ts +4 -0
- package/src/core/llm.ts +29 -0
- package/src/core/main-agent-types.ts +53 -4
- package/src/core/main-agent.ts +228 -125
- package/src/core/manifest.ts +109 -24
- package/src/core/mcp-result.ts +56 -0
- package/src/core/meta-tools.ts +258 -110
- package/src/core/middlewares/attachment-recall.ts +502 -0
- package/src/core/middlewares/capability-gate.ts +77 -28
- package/src/core/middlewares/context-guard.ts +154 -21
- package/src/core/middlewares/index.ts +5 -0
- package/src/core/middlewares/result-cap.ts +1 -1
- package/src/core/middlewares/summarization.ts +9 -0
- package/src/core/middlewares/tool-marks.ts +11 -2
- package/src/core/middlewares/turn-boundary.ts +22 -12
- package/src/core/operating-guides.ts +126 -0
- package/src/core/plugins/skills/skills-tools.ts +6 -31
- package/src/core/plugins/skills/skills.plugin.ts +4 -3
- package/src/core/plugins/weather/weather.plugin.ts +1 -0
- package/src/core/prompt-composer.ts +188 -111
- package/src/core/read-result-tool.ts +48 -14
- package/src/core/registries.ts +13 -5
- package/src/core/state.ts +4 -3
- package/src/core/sub-agent-fallback.ts +1 -0
- package/src/core/subagent-as-tool.ts +4 -1
- package/src/core/tool-access.ts +3 -32
- package/src/core/wrap-plugin-tool.ts +2 -1
- package/src/do/contracts.ts +64 -24
- package/src/do/room-mirror.ts +144 -20
- package/src/do/transcript.ts +41 -12
- package/src/do/turn-attachments.ts +45 -6
- package/src/do/turn-metadata.ts +16 -4
- package/src/do/user-oracle-do.ts +639 -152
- package/src/index.ts +1 -0
- package/src/llm/provider-error.ts +5 -2
- package/src/llm/stream-liveness.ts +3 -2
- package/src/matrix/gateway-do.ts +164 -102
- package/src/matrix/ingest.ts +12 -3
- package/src/matrix/media-deadline.ts +5 -47
- package/src/memory/session-history-indexer.ts +60 -14
- package/src/plugin-api/oracle-plugin.ts +4 -5
- package/src/plugin-api/types.ts +32 -3
- package/src/plugins/action-proposals/action-proposals.plugin.ts +7 -2
- package/src/plugins/agui/agui.plugin.ts +1 -0
- package/src/plugins/attachments/attachments.plugin.ts +3 -3
- package/src/plugins/attachments/view-attachment-tool.ts +6 -4
- package/src/plugins/compact-schema.ts +80 -0
- package/src/plugins/composio/composio.plugin.ts +1 -0
- package/src/plugins/domain-indexer/domain-indexer.plugin.ts +4 -3
- package/src/plugins/editor/editor.plugin.ts +4 -0
- package/src/plugins/firecrawl/firecrawl-agent.ts +2 -0
- package/src/plugins/firecrawl/firecrawl.plugin.ts +1 -0
- package/src/plugins/flows/flows.plugin.ts +37 -6
- package/src/plugins/flows/prompts.ts +1 -1
- package/src/plugins/flows/tools/authoring.ts +5 -7
- package/src/plugins/flows/tools/discovery.ts +1 -4
- package/src/plugins/flows/tools/forms.ts +2 -8
- package/src/plugins/flows/tools/linkage.ts +1 -4
- package/src/plugins/flows/tools/settings.ts +1 -4
- package/src/plugins/flows/translator.ts +16 -5
- package/src/plugins/flows/types.ts +27 -48
- package/src/plugins/hub-actions/hub-actions.plugin.ts +30 -4
- package/src/plugins/index.ts +2 -2
- package/src/plugins/ixo-chain/ixo-chain.plugin.ts +29 -5
- package/src/plugins/ixo-chain/kinds/core.ts +2 -2
- package/src/plugins/ixo-chain/kinds/types.ts +34 -5
- package/src/plugins/ixo-chain/queries.ts +10 -1
- package/src/plugins/ixo-transaction/ixo-transaction.plugin.ts +1 -0
- package/src/plugins/matrix-group-chats/index.ts +1 -0
- package/src/plugins/memory/memory-tools.ts +4 -4
- package/src/plugins/memory/memory.plugin.ts +27 -1
- package/src/plugins/memory/oracle-document-tools.ts +24 -49
- package/src/plugins/pod-creator/pod-creator.plugin.ts +1 -0
- package/src/plugins/portal/portal.plugin.ts +1 -0
- package/src/plugins/sandbox/sandbox-write-blob.ts +3 -15
- package/src/plugins/sandbox/sandbox.plugin.ts +19 -2
- package/src/plugins/tasks/manifest.ts +1 -0
- package/src/plugins/tasks/tasks-tools.ts +14 -19
- package/src/plugins/user-preferences/user-preferences-tool.ts +11 -37
- package/src/plugins/user-preferences/user-preferences.plugin.ts +4 -0
- package/src/plugins/vfs/vfs-tools.ts +6 -6
- package/src/plugins/vfs/vfs.plugin.ts +4 -5
- package/src/plugins/yoma/yoma.plugin.ts +1 -0
- package/src/sqlite/inline-payloads.ts +121 -0
- package/src/sqlite/serialization.ts +12 -0
- package/src/sqlite/sqlite-saver.ts +258 -42
- package/src/tasks/schedule-input.ts +5 -22
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The checkpointer's storage form of a value: every human message that
|
|
3
|
+
* carries inline attachment bytes (base64 content blocks, see
|
|
4
|
+
* `src/attachments/retention.ts`) is replaced by its placeholder copy
|
|
5
|
+
* (`offloadInlinePayloads`), wherever it sits in the value.
|
|
6
|
+
*
|
|
7
|
+
* Every byte the saver writes passes through here: message rows
|
|
8
|
+
* (`encodeMessageRow`, which `put` and `appendTurnMessages` use), the
|
|
9
|
+
* checkpoint blob's other channel values and every pending write
|
|
10
|
+
* (`putWrites` — a node's output and LangGraph's `Send` packets, which
|
|
11
|
+
* carry a copy of the whole state, messages included). The graph's
|
|
12
|
+
* in-memory state keeps the real blocks: nothing here mutates its input,
|
|
13
|
+
* and a container is copied only when something inside it was replaced,
|
|
14
|
+
* so a value without inline payloads comes back as the same object.
|
|
15
|
+
*
|
|
16
|
+
* Only human messages are replaced: a tool or AI message that carries
|
|
17
|
+
* base64 (an MCP screenshot) is stored as it is, since nothing could bring
|
|
18
|
+
* its bytes back. The saver scans what it serializes for the payload text
|
|
19
|
+
* (`containsInlinePayload`) and logs a write that still holds one, so a
|
|
20
|
+
* shape this walk misses is visible.
|
|
21
|
+
*/
|
|
22
|
+
import { isBaseMessage } from '@langchain/core/messages';
|
|
23
|
+
import { offloadInlinePayloads } from '../attachments/retention';
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* How deep a value is searched. A `Send` packet holds the state, the state
|
|
27
|
+
* holds the message list, the list holds the message: four levels; the
|
|
28
|
+
* margin covers state channels that nest a little deeper.
|
|
29
|
+
*/
|
|
30
|
+
const MAX_DEPTH = 12;
|
|
31
|
+
|
|
32
|
+
function isPlainObject(value: object): value is Record<string, unknown> {
|
|
33
|
+
const proto: unknown = Object.getPrototypeOf(value);
|
|
34
|
+
return proto === Object.prototype || proto === null;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* A LangGraph `Send` packet (`lg_name: 'Send'`), the shape the serializer
|
|
39
|
+
* recognises (`serialization.ts` `_default`, `JsonPlusSerializer`).
|
|
40
|
+
*/
|
|
41
|
+
function isSendPacket(
|
|
42
|
+
value: object,
|
|
43
|
+
): value is { lg_name: 'Send'; node: unknown; args: unknown } {
|
|
44
|
+
return Reflect.get(value, 'lg_name') === 'Send';
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
function strip(value: unknown, depth: number): unknown {
|
|
48
|
+
if (typeof value !== 'object' || value === null || depth > MAX_DEPTH)
|
|
49
|
+
return value;
|
|
50
|
+
if (isBaseMessage(value)) return offloadInlinePayloads(value) ?? value;
|
|
51
|
+
if (Array.isArray(value)) {
|
|
52
|
+
let out: unknown[] | undefined;
|
|
53
|
+
for (let i = 0; i < value.length; i++) {
|
|
54
|
+
const item: unknown = value[i];
|
|
55
|
+
const stripped = strip(item, depth + 1);
|
|
56
|
+
if (stripped === item) continue;
|
|
57
|
+
out ??= value.slice();
|
|
58
|
+
out[i] = stripped;
|
|
59
|
+
}
|
|
60
|
+
return out ?? value;
|
|
61
|
+
}
|
|
62
|
+
if (isSendPacket(value)) {
|
|
63
|
+
const args = strip(value.args, depth + 1);
|
|
64
|
+
if (args === value.args) return value;
|
|
65
|
+
// Serialized exactly like the `Send` it replaces: both serializers
|
|
66
|
+
// write `{ node, args }` (plus `timeout` when set) for this shape.
|
|
67
|
+
const timeout: unknown = Reflect.get(value, 'timeout');
|
|
68
|
+
return {
|
|
69
|
+
lg_name: 'Send',
|
|
70
|
+
node: value.node,
|
|
71
|
+
args,
|
|
72
|
+
...(timeout !== undefined ? { timeout } : {}),
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
if (!isPlainObject(value)) return value;
|
|
76
|
+
let out: Record<string, unknown> | undefined;
|
|
77
|
+
for (const [key, item] of Object.entries(value)) {
|
|
78
|
+
const stripped = strip(item, depth + 1);
|
|
79
|
+
if (stripped === item) continue;
|
|
80
|
+
out ??= { ...value };
|
|
81
|
+
out[key] = stripped;
|
|
82
|
+
}
|
|
83
|
+
return out ?? value;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* `value` as the checkpointer stores it: inline attachment payloads
|
|
88
|
+
* replaced by their placeholder (see the module comment). The same object
|
|
89
|
+
* when there is nothing to replace.
|
|
90
|
+
*/
|
|
91
|
+
export function withoutInlinePayloads(value: unknown): unknown {
|
|
92
|
+
return strip(value, 0);
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* The text every serialized inline payload block contains (`JSON.stringify`
|
|
97
|
+
* writes no spaces): what the stored-form checks and the one-time sweep of
|
|
98
|
+
* old rows look for. User text quoting it matches too, harmlessly.
|
|
99
|
+
*/
|
|
100
|
+
export const INLINE_PAYLOAD_NEEDLE = '"source_type":"base64"';
|
|
101
|
+
|
|
102
|
+
const NEEDLE_BYTES = new TextEncoder().encode(INLINE_PAYLOAD_NEEDLE);
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* Whether serialized bytes still contain an inline payload block: a shape
|
|
106
|
+
* `withoutInlinePayloads` does not reach (a plain `{ role, content }`
|
|
107
|
+
* object, a `Map` or class instance, a value nested deeper than
|
|
108
|
+
* `MAX_DEPTH`) or a non-human message carrying base64 (tool output, never
|
|
109
|
+
* stripped). One linear scan, no decoding.
|
|
110
|
+
*/
|
|
111
|
+
export function containsInlinePayload(bytes: Uint8Array): boolean {
|
|
112
|
+
const first = NEEDLE_BYTES[0]!;
|
|
113
|
+
const last = bytes.length - NEEDLE_BYTES.length;
|
|
114
|
+
for (let i = bytes.indexOf(first); i !== -1 && i <= last; ) {
|
|
115
|
+
let j = 1;
|
|
116
|
+
while (j < NEEDLE_BYTES.length && bytes[i + j] === NEEDLE_BYTES[j]) j++;
|
|
117
|
+
if (j === NEEDLE_BYTES.length) return true;
|
|
118
|
+
i = bytes.indexOf(first, i + 1);
|
|
119
|
+
}
|
|
120
|
+
return false;
|
|
121
|
+
}
|
|
@@ -12,6 +12,10 @@
|
|
|
12
12
|
* and LangGraph `Send` objects. `stringify` is fast-safe-stringify's
|
|
13
13
|
* circular-safe variant, typed.
|
|
14
14
|
*/
|
|
15
|
+
import {
|
|
16
|
+
TURN_GUIDE_NOTE_KWARG,
|
|
17
|
+
USAGE_CARRIED_KWARG,
|
|
18
|
+
} from '../core/operating-guides';
|
|
15
19
|
import { TURN_TIME_NOTE_KWARG } from '../core/turn-time-note';
|
|
16
20
|
import {
|
|
17
21
|
CHANNEL_DELIVERY_NOTICE_KWARG,
|
|
@@ -382,6 +386,14 @@ export function cleanAdditionalKwargs(
|
|
|
382
386
|
const deliveryNotice = additionalKwargs[CHANNEL_DELIVERY_NOTICE_KWARG];
|
|
383
387
|
if (typeof deliveryNotice === 'string')
|
|
384
388
|
cleaned[CHANNEL_DELIVERY_NOTICE_KWARG] = deliveryNotice;
|
|
389
|
+
const guideNote = additionalKwargs[TURN_GUIDE_NOTE_KWARG];
|
|
390
|
+
if (typeof guideNote === 'string') cleaned[TURN_GUIDE_NOTE_KWARG] = guideNote;
|
|
391
|
+
// Kept for the context guard and the summary (core/operating-guides.ts).
|
|
392
|
+
const usage = additionalKwargs[USAGE_CARRIED_KWARG];
|
|
393
|
+
if (Array.isArray(usage))
|
|
394
|
+
cleaned[USAGE_CARRIED_KWARG] = usage.filter(
|
|
395
|
+
(name): name is string => typeof name === 'string',
|
|
396
|
+
);
|
|
385
397
|
// Channel provenance, read back typed by the transcript (ChannelOrigin,
|
|
386
398
|
// ChannelReplyReference); anything malformed is dropped there.
|
|
387
399
|
for (const key of ['org.ixo.qi.origin', 'org.ixo.qi.reply'] as const)
|
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
* messages(thread_id, checkpoint_ns, checkpoint_id, message_id PK,
|
|
15
15
|
* message_type, message_content, message BLOB, created_at)
|
|
16
16
|
* schema_migrations(version PK, name, applied_at)
|
|
17
|
+
* attachment_payload_sweeps(thread_id PK, swept_at)
|
|
17
18
|
*
|
|
18
19
|
* Behaviour kept from the original:
|
|
19
20
|
* - `channel_values.messages` is stripped from the stored checkpoint and each
|
|
@@ -25,6 +26,18 @@
|
|
|
25
26
|
* - `pending_sends` migration for pre-v4 checkpoints.
|
|
26
27
|
* - Serialization is `JsonPlusSerializer` (the `BaseCheckpointSaver` default).
|
|
27
28
|
*
|
|
29
|
+
* Attachment bytes are never stored: every write path (message rows, the
|
|
30
|
+
* checkpoint blob's channel values, pending writes) stores a human message
|
|
31
|
+
* that carries inline base64 blocks as its placeholder copy
|
|
32
|
+
* (`inline-payloads.ts`), while the run's in-memory state keeps the blocks.
|
|
33
|
+
* Whatever reads the database back — the next turn, a durable run resumed
|
|
34
|
+
* after a reset, the transcript, the owner copy — sees the placeholder.
|
|
35
|
+
* Human message rows written before that (with their bytes) are rewritten
|
|
36
|
+
* once per thread, on the thread's first load (`sweepInlinePayloads`; the
|
|
37
|
+
* thread is then recorded in `attachment_payload_sweeps`). A write whose
|
|
38
|
+
* serialized form still holds a payload block is logged
|
|
39
|
+
* (`containsInlinePayload`): tool and AI messages are never stripped.
|
|
40
|
+
*
|
|
28
41
|
* Differences: everything is async (wa-sqlite API), `console` replaces the
|
|
29
42
|
* `@ixo/logger` Logger, the oracle name stamped on messages comes from
|
|
30
43
|
* `options.oracleName` instead of `process.env`, and each `put`/`putWrites`/
|
|
@@ -54,6 +67,15 @@ import {
|
|
|
54
67
|
stringify,
|
|
55
68
|
type CleanAdditionalKwargs,
|
|
56
69
|
} from './serialization';
|
|
70
|
+
import {
|
|
71
|
+
containsInlinePayload,
|
|
72
|
+
INLINE_PAYLOAD_NEEDLE,
|
|
73
|
+
withoutInlinePayloads,
|
|
74
|
+
} from './inline-payloads';
|
|
75
|
+
import {
|
|
76
|
+
hasInlinePayload,
|
|
77
|
+
offloadInlinePayloads,
|
|
78
|
+
} from '../attachments/retention';
|
|
57
79
|
|
|
58
80
|
export interface SqliteSaverOptions {
|
|
59
81
|
/**
|
|
@@ -145,6 +167,17 @@ function sameFingerprint(
|
|
|
145
167
|
);
|
|
146
168
|
}
|
|
147
169
|
|
|
170
|
+
/**
|
|
171
|
+
* The stored form of one message object: the fingerprint it was encoded
|
|
172
|
+
* from, its blob and, when the stored form is a placeholder copy (inline
|
|
173
|
+
* attachment bytes left out), that copy's `message_content` text.
|
|
174
|
+
*/
|
|
175
|
+
interface EncodedMessage {
|
|
176
|
+
fingerprint: MessageFingerprint;
|
|
177
|
+
blob: Uint8Array;
|
|
178
|
+
storedContent?: string;
|
|
179
|
+
}
|
|
180
|
+
|
|
148
181
|
/** A message's row parameters, and whether its stored row already holds them. */
|
|
149
182
|
interface EncodedMessageRow {
|
|
150
183
|
row: SqlParam[];
|
|
@@ -270,7 +303,31 @@ const migration002: Migration = {
|
|
|
270
303
|
},
|
|
271
304
|
};
|
|
272
305
|
|
|
273
|
-
|
|
306
|
+
/**
|
|
307
|
+
* Migration 003: `attachment_payload_sweeps`, the threads whose human
|
|
308
|
+
* message rows no longer hold inline attachment bytes (see
|
|
309
|
+
* `sweepInlinePayloads`).
|
|
310
|
+
*/
|
|
311
|
+
const migration003: Migration = {
|
|
312
|
+
version: 3,
|
|
313
|
+
name: 'add_attachment_payload_sweeps',
|
|
314
|
+
up: async (db) => {
|
|
315
|
+
await db.run(`
|
|
316
|
+
CREATE TABLE IF NOT EXISTS attachment_payload_sweeps (
|
|
317
|
+
thread_id TEXT NOT NULL PRIMARY KEY,
|
|
318
|
+
swept_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
|
|
319
|
+
)`);
|
|
320
|
+
},
|
|
321
|
+
};
|
|
322
|
+
|
|
323
|
+
const MIGRATIONS: readonly Migration[] = [
|
|
324
|
+
migration001,
|
|
325
|
+
migration002,
|
|
326
|
+
migration003,
|
|
327
|
+
];
|
|
328
|
+
|
|
329
|
+
/** Rows the one-time attachment sweep inflates and rewrites per transaction. */
|
|
330
|
+
const SWEEP_BATCH_ROWS = 8;
|
|
274
331
|
|
|
275
332
|
// `writes.value` blobs may be gzipped, so they travel as hex() — a
|
|
276
333
|
// CAST(... AS TEXT) of compressed bytes would mangle them irreversibly.
|
|
@@ -334,10 +391,11 @@ export class SqliteSaver extends BaseCheckpointSaver {
|
|
|
334
391
|
* third one per step — on a long thread, the peak that reset a 128 MB
|
|
335
392
|
* isolate at the end of a tool turn.
|
|
336
393
|
*/
|
|
337
|
-
private readonly encoded = new WeakMap<
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
394
|
+
private readonly encoded = new WeakMap<BaseMessage, EncodedMessage>();
|
|
395
|
+
/** Threads known to hold no inline attachment bytes in their rows. */
|
|
396
|
+
private readonly sweptThreads = new Set<string>();
|
|
397
|
+
/** The sweep in progress per thread: concurrent loads share it. */
|
|
398
|
+
private readonly sweeps = new Map<string, Promise<void>>();
|
|
341
399
|
|
|
342
400
|
constructor(
|
|
343
401
|
db: DoSqliteDatabase,
|
|
@@ -470,6 +528,7 @@ export class SqliteSaver extends BaseCheckpointSaver {
|
|
|
470
528
|
*/
|
|
471
529
|
async listThreadMessages(threadId: string): Promise<BaseMessage[]> {
|
|
472
530
|
await this.setup();
|
|
531
|
+
await this.ensureSwept(threadId);
|
|
473
532
|
const rows = await this.db.exec<MessageRow>(
|
|
474
533
|
'SELECT message FROM messages WHERE thread_id = ? ORDER BY created_at ASC, rowid ASC',
|
|
475
534
|
[threadId],
|
|
@@ -477,6 +536,23 @@ export class SqliteSaver extends BaseCheckpointSaver {
|
|
|
477
536
|
return Promise.all(rows.map((row) => this.loadMessage(row.message)));
|
|
478
537
|
}
|
|
479
538
|
|
|
539
|
+
/**
|
|
540
|
+
* The thread's newest human and AI rows with stored content, newest first.
|
|
541
|
+
* `limit` bounds the rows decoded, so the read stays O(limit) in a long
|
|
542
|
+
* thread, unlike `listThreadMessages`.
|
|
543
|
+
*/
|
|
544
|
+
async listRecentThreadMessages(
|
|
545
|
+
threadId: string,
|
|
546
|
+
limit: number,
|
|
547
|
+
): Promise<BaseMessage[]> {
|
|
548
|
+
await this.setup();
|
|
549
|
+
const rows = await this.db.exec<MessageRow>(
|
|
550
|
+
`SELECT message FROM messages WHERE thread_id = ? AND message_type IN ('human', 'ai') AND message_content <> '' ORDER BY created_at DESC, rowid DESC LIMIT ?`,
|
|
551
|
+
[threadId, limit],
|
|
552
|
+
);
|
|
553
|
+
return Promise.all(rows.map((row) => this.loadMessage(row.message)));
|
|
554
|
+
}
|
|
555
|
+
|
|
480
556
|
/**
|
|
481
557
|
* Where a transcript row sits in the listing order (`created_at`, then
|
|
482
558
|
* `rowid`), or null when the thread has no row with that message id. A
|
|
@@ -514,6 +590,7 @@ export class SqliteSaver extends BaseCheckpointSaver {
|
|
|
514
590
|
},
|
|
515
591
|
): Promise<ThreadMessageRow[]> {
|
|
516
592
|
await this.setup();
|
|
593
|
+
await this.ensureSwept(threadId);
|
|
517
594
|
const older = opts.direction === 'older';
|
|
518
595
|
const cmp = older ? '<' : '>';
|
|
519
596
|
const rowCmp = opts.inclusive ? `${cmp}=` : cmp;
|
|
@@ -548,8 +625,8 @@ export class SqliteSaver extends BaseCheckpointSaver {
|
|
|
548
625
|
/**
|
|
549
626
|
* The thread's messages whose stored plain content (`message_content`)
|
|
550
627
|
* contains `contentContains`, oldest first — a SQL `LIKE` pre-filter, so
|
|
551
|
-
* only the matching rows' blobs are inflated.
|
|
552
|
-
*
|
|
628
|
+
* only the matching rows' blobs are inflated. `view_attachment` uses it to
|
|
629
|
+
* find the messages that quote an attachment reference.
|
|
553
630
|
*/
|
|
554
631
|
async listThreadMessagesMatching(
|
|
555
632
|
threadId: string,
|
|
@@ -589,25 +666,103 @@ export class SqliteSaver extends BaseCheckpointSaver {
|
|
|
589
666
|
}
|
|
590
667
|
|
|
591
668
|
/**
|
|
592
|
-
*
|
|
593
|
-
*
|
|
594
|
-
*
|
|
595
|
-
* uses it to swap inline bytes for a placeholder after a turn.
|
|
669
|
+
* Run the thread's one-time attachment sweep unless it ran already (in
|
|
670
|
+
* this object, or recorded in `attachment_payload_sweeps`). A failed
|
|
671
|
+
* sweep is logged and left for the next load; the read goes on.
|
|
596
672
|
*/
|
|
597
|
-
|
|
598
|
-
threadId
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
673
|
+
private ensureSwept(threadId: string): Promise<void> {
|
|
674
|
+
if (this.sweptThreads.has(threadId)) return Promise.resolve();
|
|
675
|
+
const running = this.sweeps.get(threadId);
|
|
676
|
+
if (running) return running;
|
|
677
|
+
const sweep = this.sweepInlinePayloads(threadId)
|
|
678
|
+
.then(
|
|
679
|
+
() => {
|
|
680
|
+
this.sweptThreads.add(threadId);
|
|
681
|
+
},
|
|
682
|
+
(error: unknown) => {
|
|
683
|
+
console.warn(
|
|
684
|
+
`[sqlite-saver] thread ${threadId}: attachment sweep failed, retried on the next load`,
|
|
685
|
+
error,
|
|
686
|
+
);
|
|
687
|
+
},
|
|
688
|
+
)
|
|
689
|
+
.finally(() => {
|
|
690
|
+
this.sweeps.delete(threadId);
|
|
691
|
+
});
|
|
692
|
+
this.sweeps.set(threadId, sweep);
|
|
693
|
+
return sweep;
|
|
694
|
+
}
|
|
695
|
+
|
|
696
|
+
/**
|
|
697
|
+
* Rewrite the thread's human message rows that still hold inline
|
|
698
|
+
* attachment bytes (written before the saver left them out) as their
|
|
699
|
+
* placeholder copy, then record the thread as swept. Rows summarised out
|
|
700
|
+
* of graph state are never written again, so without this their bytes
|
|
701
|
+
* would stay in the transcript and the owner copy for good. The matching
|
|
702
|
+
* rows are found by their `message_content` (no blob inflated for the
|
|
703
|
+
* rest) and rewritten `SWEEP_BATCH_ROWS` at a time, each row in place
|
|
704
|
+
* (same rowid, checkpoint linkage, `created_at` and kwargs). Tool and AI
|
|
705
|
+
* messages are left as they are (see `inline-payloads.ts`).
|
|
706
|
+
*/
|
|
707
|
+
private async sweepInlinePayloads(threadId: string): Promise<void> {
|
|
708
|
+
const swept = await this.db.get<{ found: number }>(
|
|
709
|
+
'SELECT 1 AS found FROM attachment_payload_sweeps WHERE thread_id = ?',
|
|
710
|
+
[threadId],
|
|
711
|
+
);
|
|
712
|
+
if (swept !== undefined) return;
|
|
713
|
+
const needle = likeContains(INLINE_PAYLOAD_NEEDLE);
|
|
714
|
+
const matching = await this.db.exec<{ rowid: number }>(
|
|
715
|
+
`SELECT rowid FROM messages WHERE thread_id = ? AND message_type = 'human' AND message_content LIKE ? ESCAPE '\\' ORDER BY rowid`,
|
|
716
|
+
[threadId, needle],
|
|
717
|
+
);
|
|
718
|
+
let rewritten = 0;
|
|
719
|
+
for (let i = 0; i < matching.length; i += SWEEP_BATCH_ROWS) {
|
|
720
|
+
const batch = matching.slice(i, i + SWEEP_BATCH_ROWS).map((r) => r.rowid);
|
|
721
|
+
const rows = await this.db.exec<{
|
|
722
|
+
rowid: number;
|
|
723
|
+
message_id: string;
|
|
724
|
+
message: Uint8Array | string;
|
|
725
|
+
}>(
|
|
726
|
+
'SELECT rowid, message_id, message FROM messages WHERE rowid IN (SELECT value FROM json_each(?))',
|
|
727
|
+
[JSON.stringify(batch)],
|
|
728
|
+
);
|
|
729
|
+
const updates: SqlParam[][] = [];
|
|
730
|
+
for (const row of rows) {
|
|
731
|
+
const placeholder = offloadInlinePayloads(
|
|
732
|
+
await this.loadMessage(row.message),
|
|
733
|
+
);
|
|
734
|
+
if (!placeholder) continue;
|
|
735
|
+
updates.push([
|
|
736
|
+
contentColumn(placeholder.content),
|
|
737
|
+
await compressForStorage(
|
|
738
|
+
stringify(placeholder, (_key, value) => _default(value)),
|
|
739
|
+
),
|
|
740
|
+
row.rowid,
|
|
741
|
+
row.message_id,
|
|
742
|
+
needle,
|
|
743
|
+
]);
|
|
744
|
+
}
|
|
745
|
+
if (updates.length === 0) continue;
|
|
746
|
+
await this.db.transaction(async () => {
|
|
747
|
+
for (const update of updates) {
|
|
748
|
+
// Only a row that is still the one read: same message, bytes
|
|
749
|
+
// still in it (a `put` in between stored its placeholder).
|
|
750
|
+
const result = await this.db.run(
|
|
751
|
+
`UPDATE messages SET message_content = ?, message = ? WHERE rowid = ? AND message_id = ? AND message_content LIKE ? ESCAPE '\\'`,
|
|
752
|
+
update,
|
|
753
|
+
);
|
|
754
|
+
rewritten += result.changes;
|
|
755
|
+
}
|
|
756
|
+
});
|
|
757
|
+
}
|
|
758
|
+
await this.db.run(
|
|
759
|
+
'INSERT OR IGNORE INTO attachment_payload_sweeps (thread_id) VALUES (?)',
|
|
760
|
+
[threadId],
|
|
609
761
|
);
|
|
610
|
-
|
|
762
|
+
if (matching.length > 0)
|
|
763
|
+
console.info(
|
|
764
|
+
`[sqlite-saver] thread ${threadId}: ${rewritten} of ${matching.length} stored human message(s) with inline attachment bytes rewritten as their placeholder`,
|
|
765
|
+
);
|
|
611
766
|
}
|
|
612
767
|
|
|
613
768
|
private async loadMessage(raw: Uint8Array | string): Promise<BaseMessage> {
|
|
@@ -643,6 +798,7 @@ export class SqliteSaver extends BaseCheckpointSaver {
|
|
|
643
798
|
const checkpointId: unknown = configurable.checkpoint_id;
|
|
644
799
|
if (typeof threadId !== 'string' || typeof checkpointNs !== 'string')
|
|
645
800
|
return undefined;
|
|
801
|
+
await this.ensureSwept(threadId);
|
|
646
802
|
|
|
647
803
|
const row =
|
|
648
804
|
typeof checkpointId === 'string' && checkpointId.length > 0
|
|
@@ -859,7 +1015,7 @@ export class SqliteSaver extends BaseCheckpointSaver {
|
|
|
859
1015
|
if (typeof checkpointNs !== 'string')
|
|
860
1016
|
throw new Error('"checkpoint_ns" must be a string.');
|
|
861
1017
|
|
|
862
|
-
const { checkpoint, messages } =
|
|
1018
|
+
const { checkpoint, messages } = toStoredCheckpoint(checkpointIn);
|
|
863
1019
|
const [[type1, serializedCheckpoint], [type2, serializedMetadata]] =
|
|
864
1020
|
await Promise.all([
|
|
865
1021
|
this.serde.dumpsTyped(checkpoint),
|
|
@@ -869,6 +1025,8 @@ export class SqliteSaver extends BaseCheckpointSaver {
|
|
|
869
1025
|
throw new Error(
|
|
870
1026
|
'Failed to serialized checkpoint and metadata to the same type.',
|
|
871
1027
|
);
|
|
1028
|
+
if (containsInlinePayload(serializedCheckpoint))
|
|
1029
|
+
warnStoredPayload(`the checkpoint of thread ${threadId}`);
|
|
872
1030
|
|
|
873
1031
|
const messageRows = await Promise.all(
|
|
874
1032
|
messages?.map((message) =>
|
|
@@ -1034,6 +1192,9 @@ export class SqliteSaver extends BaseCheckpointSaver {
|
|
|
1034
1192
|
*/
|
|
1035
1193
|
private rememberLoaded(message: BaseMessage, raw: Uint8Array | string): void {
|
|
1036
1194
|
if (!(raw instanceof Uint8Array)) return;
|
|
1195
|
+
// A row written before attachment bytes were left out of storage still
|
|
1196
|
+
// holds them: not remembered, so the next `put` stores its placeholder.
|
|
1197
|
+
if (hasInlinePayload(message)) return;
|
|
1037
1198
|
this.encoded.set(message, {
|
|
1038
1199
|
fingerprint: fingerprintOf(message, this.normalizeKwargs(message)),
|
|
1039
1200
|
blob: raw,
|
|
@@ -1044,6 +1205,9 @@ export class SqliteSaver extends BaseCheckpointSaver {
|
|
|
1044
1205
|
* A message's row parameters. A message whose fingerprint matches the
|
|
1045
1206
|
* blob last encoded or loaded for it reuses that blob without
|
|
1046
1207
|
* serializing it (`unchanged`); any other is serialized and compressed.
|
|
1208
|
+
* A human message with inline attachment bytes is stored as its
|
|
1209
|
+
* placeholder copy (`offloadInlinePayloads`): the copy is serialized once
|
|
1210
|
+
* and dropped, and only its blob and content text are remembered.
|
|
1047
1211
|
*/
|
|
1048
1212
|
private async encodeMessageRow(
|
|
1049
1213
|
message: BaseMessage,
|
|
@@ -1053,20 +1217,26 @@ export class SqliteSaver extends BaseCheckpointSaver {
|
|
|
1053
1217
|
): Promise<EncodedMessageRow> {
|
|
1054
1218
|
const merged = this.normalizeKwargs(message);
|
|
1055
1219
|
const fingerprint = fingerprintOf(message, merged);
|
|
1056
|
-
|
|
1057
|
-
let serialized: Uint8Array;
|
|
1220
|
+
let encoded = this.encoded.get(message);
|
|
1058
1221
|
let unchanged = false;
|
|
1059
1222
|
if (
|
|
1060
|
-
|
|
1061
|
-
sameFingerprint(
|
|
1223
|
+
encoded !== undefined &&
|
|
1224
|
+
sameFingerprint(encoded.fingerprint, fingerprint)
|
|
1062
1225
|
) {
|
|
1063
|
-
serialized = cached.blob;
|
|
1064
1226
|
unchanged = true;
|
|
1065
1227
|
} else {
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1228
|
+
const placeholder = offloadInlinePayloads(message);
|
|
1229
|
+
const stored = placeholder ?? message;
|
|
1230
|
+
encoded = {
|
|
1231
|
+
fingerprint,
|
|
1232
|
+
blob: await compressForStorage(
|
|
1233
|
+
stringify(stored, (_key, value) => _default(value)),
|
|
1234
|
+
),
|
|
1235
|
+
...(placeholder
|
|
1236
|
+
? { storedContent: contentColumn(placeholder.content) }
|
|
1237
|
+
: {}),
|
|
1238
|
+
};
|
|
1239
|
+
this.encoded.set(message, encoded);
|
|
1070
1240
|
}
|
|
1071
1241
|
const messageId =
|
|
1072
1242
|
message.id ??
|
|
@@ -1081,10 +1251,8 @@ export class SqliteSaver extends BaseCheckpointSaver {
|
|
|
1081
1251
|
checkpointId,
|
|
1082
1252
|
messageId,
|
|
1083
1253
|
message.getType(),
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
: JSON.stringify(message.content),
|
|
1087
|
-
serialized,
|
|
1254
|
+
encoded.storedContent ?? contentColumn(message.content),
|
|
1255
|
+
encoded.blob,
|
|
1088
1256
|
merged.timestamp,
|
|
1089
1257
|
],
|
|
1090
1258
|
unchanged,
|
|
@@ -1170,9 +1338,13 @@ export class SqliteSaver extends BaseCheckpointSaver {
|
|
|
1170
1338
|
const checkpointNs: unknown = configurable.checkpoint_ns;
|
|
1171
1339
|
const ns = typeof checkpointNs === 'string' ? checkpointNs : '';
|
|
1172
1340
|
|
|
1341
|
+
const holding: string[] = [];
|
|
1173
1342
|
const rows = await Promise.all(
|
|
1174
1343
|
writes.map(async (write, idx): Promise<SqlParam[]> => {
|
|
1175
|
-
const [type, serialized] = await this.serde.dumpsTyped(
|
|
1344
|
+
const [type, serialized] = await this.serde.dumpsTyped(
|
|
1345
|
+
withoutInlinePayloads(write[1]),
|
|
1346
|
+
);
|
|
1347
|
+
if (containsInlinePayload(serialized)) holding.push(write[0]);
|
|
1176
1348
|
return [
|
|
1177
1349
|
resolvedThreadId,
|
|
1178
1350
|
ns,
|
|
@@ -1185,6 +1357,10 @@ export class SqliteSaver extends BaseCheckpointSaver {
|
|
|
1185
1357
|
];
|
|
1186
1358
|
}),
|
|
1187
1359
|
);
|
|
1360
|
+
if (holding.length > 0)
|
|
1361
|
+
warnStoredPayload(
|
|
1362
|
+
`a pending write of thread ${resolvedThreadId} (task ${taskId}, channel ${holding.join(', ')})`,
|
|
1363
|
+
);
|
|
1188
1364
|
await this.db.transaction(async () => {
|
|
1189
1365
|
for (const row of rows) {
|
|
1190
1366
|
await this.db.run(
|
|
@@ -1198,7 +1374,12 @@ export class SqliteSaver extends BaseCheckpointSaver {
|
|
|
1198
1374
|
|
|
1199
1375
|
async deleteThread(threadId: string): Promise<void> {
|
|
1200
1376
|
await this.setup();
|
|
1377
|
+
this.sweptThreads.delete(threadId);
|
|
1201
1378
|
await this.db.transaction(async () => {
|
|
1379
|
+
await this.db.run(
|
|
1380
|
+
'DELETE FROM attachment_payload_sweeps WHERE thread_id = ?',
|
|
1381
|
+
[threadId],
|
|
1382
|
+
);
|
|
1202
1383
|
await this.db.run('DELETE FROM checkpoints WHERE thread_id = ?', [
|
|
1203
1384
|
threadId,
|
|
1204
1385
|
]);
|
|
@@ -1253,18 +1434,53 @@ function asReasoningDetails(
|
|
|
1253
1434
|
);
|
|
1254
1435
|
}
|
|
1255
1436
|
|
|
1437
|
+
/**
|
|
1438
|
+
* One line for a write that stores inline base64 after stripping
|
|
1439
|
+
* (`containsInlinePayload`): a tool or AI message carrying base64, or a
|
|
1440
|
+
* human message in a shape `withoutInlinePayloads` does not reach.
|
|
1441
|
+
*/
|
|
1442
|
+
function warnStoredPayload(what: string): void {
|
|
1443
|
+
console.warn(
|
|
1444
|
+
`[sqlite-saver] ${what} still holds inline base64 after attachment stripping: a tool/AI message with base64 content, or a message shape inline-payloads.ts does not reach; those bytes are stored`,
|
|
1445
|
+
);
|
|
1446
|
+
}
|
|
1447
|
+
|
|
1448
|
+
/** The `messages.message_content` text of a message's content. */
|
|
1449
|
+
function contentColumn(content: BaseMessage['content']): string {
|
|
1450
|
+
return typeof content === 'string' ? content : JSON.stringify(content);
|
|
1451
|
+
}
|
|
1452
|
+
|
|
1256
1453
|
function isCheckpointWithMessages(
|
|
1257
1454
|
checkpoint: Checkpoint,
|
|
1258
1455
|
): checkpoint is CheckpointWithMessages {
|
|
1259
1456
|
return 'messages' in checkpoint.channel_values;
|
|
1260
1457
|
}
|
|
1261
1458
|
|
|
1262
|
-
|
|
1459
|
+
/**
|
|
1460
|
+
* The checkpoint as its blob stores it: `channel_values.messages` taken out
|
|
1461
|
+
* (they become message rows, returned) and inline attachment bytes left out
|
|
1462
|
+
* of every other channel value. The input is never modified; it is returned
|
|
1463
|
+
* as is when there is nothing to take out.
|
|
1464
|
+
*/
|
|
1465
|
+
function toStoredCheckpoint(checkpoint: Checkpoint): {
|
|
1263
1466
|
checkpoint: Checkpoint;
|
|
1264
1467
|
messages?: BaseMessage[];
|
|
1265
1468
|
} {
|
|
1266
|
-
|
|
1469
|
+
const messages = isCheckpointWithMessages(checkpoint)
|
|
1470
|
+
? checkpoint.channel_values.messages
|
|
1471
|
+
: undefined;
|
|
1472
|
+
let channelValues: ChannelValues | undefined;
|
|
1473
|
+
for (const [channel, value] of Object.entries(checkpoint.channel_values)) {
|
|
1474
|
+
if (channel === 'messages') continue;
|
|
1475
|
+
const stored = withoutInlinePayloads(value);
|
|
1476
|
+
if (stored === value) continue;
|
|
1477
|
+
channelValues ??= { ...checkpoint.channel_values };
|
|
1478
|
+
channelValues[channel] = stored;
|
|
1479
|
+
}
|
|
1480
|
+
if (messages === undefined && channelValues === undefined)
|
|
1481
|
+
return { checkpoint };
|
|
1267
1482
|
const copy = copyCheckpoint(checkpoint);
|
|
1483
|
+
if (channelValues !== undefined) copy.channel_values = channelValues;
|
|
1268
1484
|
delete copy.channel_values.messages;
|
|
1269
|
-
return { checkpoint: copy, messages
|
|
1485
|
+
return { checkpoint: copy, ...(messages ? { messages } : {}) };
|
|
1270
1486
|
}
|
|
@@ -14,32 +14,15 @@ import { z } from 'zod';
|
|
|
14
14
|
import type { OracleTaskSchedule } from '../plugin-api/types';
|
|
15
15
|
|
|
16
16
|
const SCHEDULE_DESCRIPTION =
|
|
17
|
-
'
|
|
17
|
+
'kind "once" needs `at`, computed from the current time: ALWAYS for one-time requests such as "in 10 minutes" or "tomorrow at 5pm". kind "cron" needs `cron` and is ONLY for genuinely recurring intents: `*/10 * * * *` means every 10 minutes forever, not once in 10 minutes. kind "interval" needs `everySeconds`. Leave the other kinds\' fields out.';
|
|
18
18
|
|
|
19
19
|
export const ScheduleInputSchema = z
|
|
20
20
|
.object({
|
|
21
21
|
kind: z.enum(['once', 'cron', 'interval']),
|
|
22
|
-
at: z
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
.describe('ISO timestamp — kind "once" only.'),
|
|
27
|
-
cron: z
|
|
28
|
-
.string()
|
|
29
|
-
.min(1)
|
|
30
|
-
.optional()
|
|
31
|
-
.describe('Cron expression (5 fields) — kind "cron" only.'),
|
|
32
|
-
timezone: z
|
|
33
|
-
.string()
|
|
34
|
-
.min(1)
|
|
35
|
-
.optional()
|
|
36
|
-
.describe('IANA timezone for the cron expression — kind "cron" only.'),
|
|
37
|
-
everySeconds: z
|
|
38
|
-
.number()
|
|
39
|
-
.int()
|
|
40
|
-
.positive()
|
|
41
|
-
.optional()
|
|
42
|
-
.describe('Seconds between runs — kind "interval" only.'),
|
|
22
|
+
at: z.string().min(1).optional().describe('ISO timestamp.'),
|
|
23
|
+
cron: z.string().min(1).optional().describe('5 fields.'),
|
|
24
|
+
timezone: z.string().min(1).optional().describe('IANA, for `cron`.'),
|
|
25
|
+
everySeconds: z.number().int().positive().optional(),
|
|
43
26
|
})
|
|
44
27
|
.describe(SCHEDULE_DESCRIPTION);
|
|
45
28
|
|