@sema-agent/server 7.93.6 → 7.94.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/USAGE.md +8 -0
- package/dist/boot/stores.js +6 -3
- package/dist/boot/workflow-orchestration.js +5 -1
- package/dist/config-catalog.js +1 -1
- package/dist/http/resume-ctx-read.d.ts +40 -0
- package/dist/http/resume-ctx-read.js +25 -0
- package/dist/http/resume-legs.js +24 -14
- package/dist/http/routes/images.js +12 -0
- package/dist/http/routes/runs.js +18 -4
- package/dist/http/routes/workflows.js +3 -2
- package/dist/observability/corrupt-read-seat.d.ts +56 -0
- package/dist/observability/corrupt-read-seat.js +16 -0
- package/dist/observability/fail-open.d.ts +4 -0
- package/dist/observability/fail-open.js +4 -0
- package/dist/orchestration/workflow-journal-entry.d.ts +24 -6
- package/dist/orchestration/workflow-journal-entry.js +17 -10
- package/dist/plugins/approval-ask-store-sql.js +2 -2
- package/dist/plugins/checkpoint-store-sql.d.ts +16 -3
- package/dist/plugins/checkpoint-store-sql.js +7 -9
- package/dist/plugins/image-bake-store-sql.js +85 -13
- package/dist/plugins/image-index-sql.js +75 -12
- package/dist/plugins/local-checkpoint-store.d.ts +23 -2
- package/dist/plugins/local-checkpoint-store.js +40 -33
- package/dist/plugins/local-session-store.d.ts +14 -0
- package/dist/plugins/local-session-store.js +12 -7
- package/dist/plugins/local-sidecar-quarantine.d.ts +10 -0
- package/dist/plugins/local-sidecar-quarantine.js +15 -0
- package/dist/plugins/local-task-attachment-store.d.ts +1 -1
- package/dist/plugins/local-task-attachment-store.js +5 -2
- package/dist/plugins/mailbox-store-sql.d.ts +6 -3
- package/dist/plugins/mailbox-store-sql.js +45 -43
- package/dist/plugins/memory-embedder-fingerprint.js +8 -12
- package/dist/plugins/memory-engine-pg.js +34 -16
- package/dist/plugins/memory-engine-tidb.js +19 -12
- package/dist/plugins/memory-engine-vector-util.d.ts +26 -0
- package/dist/plugins/memory-engine-vector-util.js +3 -0
- package/dist/plugins/memory-sync-store-pg.d.ts +6 -0
- package/dist/plugins/memory-sync-store-pg.js +14 -10
- package/dist/plugins/memory-sync-store-tidb.js +7 -11
- package/dist/plugins/pg-session-storage.js +26 -28
- package/dist/plugins/session-event-payload.d.ts +37 -0
- package/dist/plugins/session-event-payload.js +20 -0
- package/dist/plugins/sql-json-column.d.ts +124 -22
- package/dist/plugins/sql-json-column.js +22 -1
- package/dist/plugins/sql-row-helpers.d.ts +10 -16
- package/dist/plugins/sql-row-helpers.js +0 -29
- package/dist/plugins/store-backend.js +10 -12
- package/dist/plugins/store-contracts.d.ts +4 -6
- package/dist/plugins/store-contracts.js +0 -60
- package/dist/plugins/task-attachment-store.d.ts +21 -12
- package/dist/plugins/task-attachment-store.js +12 -0
- package/dist/plugins/tidb-session-storage.d.ts +2 -2
- package/dist/plugins/tidb-session-storage.js +12 -11
- package/dist/plugins/tidb-session-store.d.ts +2 -2
- package/dist/plugins/tidb-session-store.js +20 -22
- package/dist/plugins/usage-window-store-sql.d.ts +9 -1
- package/dist/plugins/usage-window-store-sql.js +23 -9
- package/dist/plugins/workflow-journal-store-sql.js +16 -4
- package/dist/run-local.js +2 -10
- package/dist/usage-analytics.d.ts +14 -1
- package/dist/usage-analytics.js +11 -17
- package/package.json +2 -2
|
@@ -55,9 +55,12 @@ export declare const MAILBOX_MSG_TABLE = "mailbox_message";
|
|
|
55
55
|
* —— `mailbox` 行是**盒**事实(next_seq / lease),墓碑是**收件人**事实,两者的生命周期本来就不同
|
|
56
56
|
* (`drop` 同时终结两者 —— 墓碑随箱亡,理由全文见 `mailbox-recipient-lifecycle.ts` 模块头)。 */
|
|
57
57
|
export declare const MAILBOX_TOMBSTONE_TABLE = "mailbox_tombstone";
|
|
58
|
-
/** 店侧日志座(可选;boot 注入 `Logger`)。7.58.0 合并重扫 [medium]
|
|
59
|
-
* claimLease 调用方(peer-session-drain)吞掉该错,店不记这一笔就成了「静默永久卡箱」。抛之前记
|
|
60
|
-
* (scope/handle/seq/code),运维按 seq 修行。
|
|
58
|
+
/** 店侧日志座(可选;boot 注入 `Logger`)。7.58.0 合并重扫 [medium]:坏行的 fail-closed 抛必须**响亮**——core 唯一的
|
|
59
|
+
* claimLease 调用方(peer-session-drain)吞掉该错,店不记这一笔就成了「静默永久卡箱」。抛之前记
|
|
60
|
+
* `mailbox_message_column_corrupt`(scope/handle/seq/column/code),运维按 seq 修行。
|
|
61
|
+
* 🔴 S-550 车③a **改名并归一**:旧事件名 `mailbox_peer_meta_corrupt` 在 `hop_chain` 也变成 fail-closed 的
|
|
62
|
+
* 那一刻就说不出真话了(同一条读腿两列同判)。一个事件 + 一个 `column` 字段,而不是两个事件名 —— 运维
|
|
63
|
+
* 面按 column 分桶,店里只有一条留痕规则。 */
|
|
61
64
|
export interface MailboxStoreLogger {
|
|
62
65
|
warn(event: string, fields: Record<string, unknown>): void;
|
|
63
66
|
}
|
|
@@ -2,6 +2,7 @@ import { MAILBOX_INVALID_PEER_META_CODE, MAILBOX_TOMBSTONED_RECIPIENT_CODE, Mail
|
|
|
2
2
|
import { pgHasUnstorable, pgSanitizeText, PgUnstorableError } from "./pg-safe-json.js";
|
|
3
3
|
import { ensureIndexes } from "./ensure-index.js";
|
|
4
4
|
import { escapeLike } from "./sql-escape.js";
|
|
5
|
+
import { decodeNullableJsonColumn, foldJsonColumn, jsonColumnText, readStringArrayColumn } from "./sql-json-column.js";
|
|
5
6
|
export const MAILBOX_TABLE = "mailbox";
|
|
6
7
|
export const MAILBOX_MSG_TABLE = "mailbox_message";
|
|
7
8
|
export const MAILBOX_TOMBSTONE_TABLE = "mailbox_tombstone";
|
|
@@ -46,19 +47,13 @@ function encodeHopChain(hopChain) {
|
|
|
46
47
|
}
|
|
47
48
|
return text;
|
|
48
49
|
}
|
|
49
|
-
function decodeHopChain(raw) {
|
|
50
|
-
if (raw === null || raw === undefined)
|
|
51
|
-
return undefined;
|
|
52
|
-
let parsed;
|
|
50
|
+
function decodeHopChain(raw, seq) {
|
|
53
51
|
try {
|
|
54
|
-
|
|
52
|
+
return decodeNullableJsonColumn(raw, readStringArrayColumn, `${MAILBOX_MSG_TABLE}.hop_chain for seq ${String(seq)}`);
|
|
55
53
|
}
|
|
56
|
-
catch {
|
|
57
|
-
|
|
54
|
+
catch (e) {
|
|
55
|
+
throw foldJsonColumn(e, (m) => new Error(`MailboxStore: ${m} — refusing the lease rather than let a message skip peer admission (an absent chain makes core's judgePeerAdmission not run at all)`));
|
|
58
56
|
}
|
|
59
|
-
if (!Array.isArray(parsed) || parsed.some((x) => typeof x !== "string"))
|
|
60
|
-
return undefined;
|
|
61
|
-
return parsed;
|
|
62
57
|
}
|
|
63
58
|
const PEER_META_MAX_BYTES = 60_000;
|
|
64
59
|
function encodePeerMeta(raw) {
|
|
@@ -71,52 +66,59 @@ function encodePeerMeta(raw) {
|
|
|
71
66
|
}
|
|
72
67
|
return text;
|
|
73
68
|
}
|
|
74
|
-
function decodePeerMeta(raw) {
|
|
75
|
-
if (raw === null || raw === undefined)
|
|
76
|
-
return undefined;
|
|
77
|
-
const text = bufToStr(raw);
|
|
78
|
-
let parsed;
|
|
69
|
+
function decodePeerMeta(raw, seq) {
|
|
79
70
|
try {
|
|
80
|
-
|
|
71
|
+
return decodeNullableJsonColumn(raw, readPeerMetaColumn, `${MAILBOX_MSG_TABLE}.peer_meta for seq ${String(seq)}`);
|
|
81
72
|
}
|
|
82
|
-
catch (
|
|
83
|
-
|
|
73
|
+
catch (e) {
|
|
74
|
+
if (e instanceof MailboxStoreError)
|
|
75
|
+
throw e;
|
|
76
|
+
throw foldJsonColumn(e, (m) => new MailboxStoreError(MAILBOX_INVALID_PEER_META_CODE, `MailboxStore: ${m} — refusing the lease rather than read a corrupt typed record as "no metadata"`));
|
|
84
77
|
}
|
|
78
|
+
}
|
|
79
|
+
const readPeerMetaColumn = (body) => {
|
|
85
80
|
try {
|
|
86
|
-
|
|
81
|
+
const meta = readMailboxPeerMeta(body);
|
|
82
|
+
return meta === undefined ? { problem: "the JSON literal `null` — absence is SQL NULL on this column, so a written `null` can only be a hand edit" } : { value: meta };
|
|
87
83
|
}
|
|
88
84
|
catch (err) {
|
|
89
85
|
if (err instanceof MailboxStoreError)
|
|
90
86
|
throw err;
|
|
91
|
-
|
|
87
|
+
return { problem: err instanceof Error ? err.message : String(err) };
|
|
92
88
|
}
|
|
93
|
-
}
|
|
89
|
+
};
|
|
94
90
|
function rowsToMessages(rows, scope, handle, logger) {
|
|
95
91
|
const out = [];
|
|
96
92
|
for (const r of rows) {
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
logger?.warn("
|
|
103
|
-
|
|
104
|
-
|
|
93
|
+
const read = (column, decode) => {
|
|
94
|
+
try {
|
|
95
|
+
return decode();
|
|
96
|
+
}
|
|
97
|
+
catch (err) {
|
|
98
|
+
logger?.warn("mailbox_message_column_corrupt", {
|
|
99
|
+
scope,
|
|
100
|
+
handle,
|
|
101
|
+
seq: Number(r.seq),
|
|
102
|
+
column,
|
|
103
|
+
...(err instanceof MailboxStoreError ? { code: err.code } : {}),
|
|
104
|
+
note: "claimLease refused for the whole box (fail-closed); repair or NULL that column of this row",
|
|
105
|
+
});
|
|
106
|
+
throw err;
|
|
107
|
+
}
|
|
108
|
+
};
|
|
109
|
+
const hopChain = read("hop_chain", () => decodeHopChain(r.hop_chain, r.seq));
|
|
110
|
+
const peerMeta = read("peer_meta", () => decodePeerMeta(r.peer_meta, r.seq));
|
|
111
|
+
out.push({
|
|
112
|
+
seq: Number(r.seq),
|
|
113
|
+
...(r.from_id !== null && r.from_id !== undefined ? { from: bufToStr(r.from_id) } : {}),
|
|
114
|
+
content: String(r.content),
|
|
115
|
+
sentAt: Number(r.sent_at_ms),
|
|
116
|
+
...(hopChain !== undefined ? { hopChain } : {}),
|
|
117
|
+
...(peerMeta !== undefined ? { peerMeta } : {}),
|
|
118
|
+
});
|
|
105
119
|
}
|
|
106
120
|
return out;
|
|
107
121
|
}
|
|
108
|
-
function rowToMessage(r) {
|
|
109
|
-
const hopChain = decodeHopChain(r.hop_chain);
|
|
110
|
-
const peerMeta = decodePeerMeta(r.peer_meta);
|
|
111
|
-
return {
|
|
112
|
-
seq: Number(r.seq),
|
|
113
|
-
...(r.from_id !== null && r.from_id !== undefined ? { from: bufToStr(r.from_id) } : {}),
|
|
114
|
-
content: String(r.content),
|
|
115
|
-
sentAt: Number(r.sent_at_ms),
|
|
116
|
-
...(hopChain !== undefined ? { hopChain } : {}),
|
|
117
|
-
...(peerMeta !== undefined ? { peerMeta } : {}),
|
|
118
|
-
};
|
|
119
|
-
}
|
|
120
122
|
export async function ensureTiDBMailboxSchema(pool) {
|
|
121
123
|
await pool.query(`CREATE TABLE IF NOT EXISTS ${MAILBOX_TABLE} (
|
|
122
124
|
scope_key VARBINARY(190) NOT NULL,
|
|
@@ -250,7 +252,7 @@ export class TiDBMailboxStore {
|
|
|
250
252
|
bufToStr(box.lease_owner) !== owner) {
|
|
251
253
|
return null;
|
|
252
254
|
}
|
|
253
|
-
const [msgs] = (await c.query(`SELECT seq, from_id, content, sent_at_ms, hop_chain, peer_meta FROM ${MAILBOX_MSG_TABLE} WHERE scope_key = ? AND handle = ? ORDER BY seq FOR UPDATE`, [scope, handle]));
|
|
255
|
+
const [msgs] = (await c.query(`SELECT seq, from_id, content, sent_at_ms, ${jsonColumnText("tidb", "hop_chain")}, ${jsonColumnText("tidb", "peer_meta")} FROM ${MAILBOX_MSG_TABLE} WHERE scope_key = ? AND handle = ? ORDER BY seq FOR UPDATE`, [scope, handle]));
|
|
254
256
|
if (msgs.length === 0)
|
|
255
257
|
return null;
|
|
256
258
|
const maxSeq = Number(msgs[msgs.length - 1].seq);
|
|
@@ -397,7 +399,7 @@ export class PgMailboxStore {
|
|
|
397
399
|
const box = boxes[0];
|
|
398
400
|
if (box.lease_owner !== null && Number(box.lease_expires_at_ms) > now && String(box.lease_owner) !== owner)
|
|
399
401
|
return null;
|
|
400
|
-
const { rows: msgs } = await c.query(`SELECT seq, from_id, content, sent_at_ms, hop_chain, peer_meta FROM ${MAILBOX_MSG_TABLE} WHERE scope_key = $1 AND handle = $2 ORDER BY seq FOR UPDATE`, [scope, handle]);
|
|
402
|
+
const { rows: msgs } = await c.query(`SELECT seq, from_id, content, sent_at_ms, ${jsonColumnText("pg", "hop_chain")}, ${jsonColumnText("pg", "peer_meta")} FROM ${MAILBOX_MSG_TABLE} WHERE scope_key = $1 AND handle = $2 ORDER BY seq FOR UPDATE`, [scope, handle]);
|
|
401
403
|
if (msgs.length === 0)
|
|
402
404
|
return null;
|
|
403
405
|
const maxSeq = Number(msgs[msgs.length - 1].seq);
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
2
|
import { PG_MEMORY_ENGINE_TABLES } from "./memory-engine-pg.js";
|
|
3
|
+
import { createZodColumnReader, decodeJsonColumn, jsonColumnText, SqlJsonColumnError } from "./sql-json-column.js";
|
|
3
4
|
export const PG_MEMORY_ENGINE_META_TABLE = "agent_memory_engine_meta";
|
|
4
5
|
export const EMBEDDER_IDENTITY_META_KEY = "embedder_identity";
|
|
5
6
|
export const CAS_MAX_ROUNDS = 3;
|
|
@@ -13,28 +14,23 @@ export async function ensurePgMemoryEmbedderMetaSchema(query) {
|
|
|
13
14
|
meta_value jsonb NOT NULL
|
|
14
15
|
)`);
|
|
15
16
|
}
|
|
17
|
+
const readEmbedderIdentityColumn = createZodColumnReader(EmbedderIdentitySchema);
|
|
16
18
|
function parseStoredText(text) {
|
|
17
|
-
let candidate;
|
|
18
19
|
try {
|
|
19
|
-
|
|
20
|
+
return { kind: "ok", value: decodeJsonColumn(text, readEmbedderIdentityColumn, `${PG_MEMORY_ENGINE_META_TABLE}.meta_value for key ${EMBEDDER_IDENTITY_META_KEY}`) };
|
|
20
21
|
}
|
|
21
22
|
catch (err) {
|
|
22
|
-
|
|
23
|
+
if (!(err instanceof SqlJsonColumnError))
|
|
24
|
+
throw err;
|
|
25
|
+
return { kind: "unreadable", raw: `${text.slice(0, 160)} [${err.problem}: ${err.message.slice(0, 120)}]` };
|
|
23
26
|
}
|
|
24
|
-
if (typeof candidate !== "object" || candidate === null || Array.isArray(candidate)) {
|
|
25
|
-
return { kind: "unreadable", raw: `${text.slice(0, 160)} [not a JSON object]` };
|
|
26
|
-
}
|
|
27
|
-
const parsed = EmbedderIdentitySchema.safeParse(candidate);
|
|
28
|
-
if (!parsed.success)
|
|
29
|
-
return { kind: "unreadable", raw: text.slice(0, 200) };
|
|
30
|
-
return { kind: "ok", value: parsed.data };
|
|
31
27
|
}
|
|
32
28
|
async function readStored(query) {
|
|
33
|
-
const r = await query(`SELECT meta_value
|
|
29
|
+
const r = await query(`SELECT ${jsonColumnText("pg", "meta_value")} FROM ${PG_MEMORY_ENGINE_META_TABLE} WHERE meta_key = $1`, [EMBEDDER_IDENTITY_META_KEY]);
|
|
34
30
|
const row = r.rows[0];
|
|
35
31
|
if (row === undefined)
|
|
36
32
|
return { kind: "absent" };
|
|
37
|
-
const text = row.
|
|
33
|
+
const text = row.meta_value;
|
|
38
34
|
if (typeof text !== "string")
|
|
39
35
|
return { kind: "unreadable", raw: `[meta_value::text did not return text: ${typeof text}]` };
|
|
40
36
|
return parseStoredText(text);
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { cosineDistance, jaccardDistance, termSet } from "@sema-agent/core";
|
|
2
|
-
import { isUniqueViolation } from "./memory-engine-vector-util.js";
|
|
2
|
+
import { isUniqueViolation, readEmbeddingColumn, readFrontmatterColumn } from "./memory-engine-vector-util.js";
|
|
3
|
+
import { decodeJsonColumn, decodeNullableJsonColumn, foldJsonColumn, isJsonColumnBodyProblem, jsonColumnText } from "./sql-json-column.js";
|
|
4
|
+
import { recordFailOpen } from "../observability/fail-open.js";
|
|
3
5
|
import { computeEntryRev, serializeEntryFile } from "@sema-agent/core";
|
|
4
6
|
import { pgSafeJsonStringify, pgHasUnstorable } from "./pg-safe-json.js";
|
|
5
7
|
import { assertMemoryScopeWidth, memoryEntryKeyWidthRefusal } from "./memory-key-guards.js";
|
|
@@ -44,8 +46,24 @@ function vectorLiteral(v) {
|
|
|
44
46
|
function inPlaceholders(first, count) {
|
|
45
47
|
return Array.from({ length: count }, (_, i) => `$${first + i}`).join(", ");
|
|
46
48
|
}
|
|
47
|
-
function
|
|
48
|
-
|
|
49
|
+
function readFrontmatter(cell, id) {
|
|
50
|
+
try {
|
|
51
|
+
return decodeJsonColumn(cell, readFrontmatterColumn, `${PG_MEMORY_ENGINE_TABLES.entry}.frontmatter for entry ${String(id)}`);
|
|
52
|
+
}
|
|
53
|
+
catch (e) {
|
|
54
|
+
throw foldJsonColumn(e, (m) => new Error(`memory-engine: ${m} — refusing rather than serve a note whose frontmatter (the origin/exposure law carrier) nobody can read`));
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
function readEmbedding(cell, id) {
|
|
58
|
+
try {
|
|
59
|
+
return decodeNullableJsonColumn(cell, readEmbeddingColumn, `${PG_MEMORY_ENGINE_TABLES.entry}.embedding for entry ${String(id)}`);
|
|
60
|
+
}
|
|
61
|
+
catch (e) {
|
|
62
|
+
if (!isJsonColumnBodyProblem(e))
|
|
63
|
+
throw e;
|
|
64
|
+
recordFailOpen("server.sql-json.memory-engine.embedding");
|
|
65
|
+
return undefined;
|
|
66
|
+
}
|
|
49
67
|
}
|
|
50
68
|
export class PgMemoryEngineBackend {
|
|
51
69
|
query;
|
|
@@ -78,7 +96,7 @@ export class PgMemoryEngineBackend {
|
|
|
78
96
|
return this.pgvectorSql ? vectorLiteral(v) : pgSafeJsonStringify(v);
|
|
79
97
|
}
|
|
80
98
|
headerFromRow(r) {
|
|
81
|
-
const fm =
|
|
99
|
+
const fm = readFrontmatter(r.frontmatter, r.id);
|
|
82
100
|
return {
|
|
83
101
|
id: r.id,
|
|
84
102
|
slug: r.slug,
|
|
@@ -98,7 +116,7 @@ export class PgMemoryEngineBackend {
|
|
|
98
116
|
id: r.id,
|
|
99
117
|
slug: r.slug,
|
|
100
118
|
scope: r.scope,
|
|
101
|
-
frontmatter:
|
|
119
|
+
frontmatter: readFrontmatter(r.frontmatter, r.id),
|
|
102
120
|
body: r.body,
|
|
103
121
|
rev: r.rev,
|
|
104
122
|
};
|
|
@@ -107,14 +125,14 @@ export class PgMemoryEngineBackend {
|
|
|
107
125
|
const uniq = [...new Set(scopes)];
|
|
108
126
|
if (uniq.length === 0)
|
|
109
127
|
return [];
|
|
110
|
-
const res = await this.query(`SELECT id, scope, slug, frontmatter, rev, mtime_ms, size_bytes FROM ${PG_MEMORY_ENGINE_TABLES.entry} WHERE scope IN (${inPlaceholders(1, uniq.length)}) ORDER BY id`, uniq);
|
|
128
|
+
const res = await this.query(`SELECT id, scope, slug, ${jsonColumnText("pg", "frontmatter")}, rev, mtime_ms, size_bytes FROM ${PG_MEMORY_ENGINE_TABLES.entry} WHERE scope IN (${inPlaceholders(1, uniq.length)}) ORDER BY id`, uniq);
|
|
111
129
|
return res.rows.map((r) => this.headerFromRow(r));
|
|
112
130
|
}
|
|
113
131
|
async getByIds(ids) {
|
|
114
132
|
const uniq = [...new Set(ids)];
|
|
115
133
|
if (uniq.length === 0)
|
|
116
134
|
return [];
|
|
117
|
-
const res = await this.query(`SELECT id, scope, slug, frontmatter, body, rev, mtime_ms, size_bytes FROM ${PG_MEMORY_ENGINE_TABLES.entry} WHERE id IN (${inPlaceholders(1, uniq.length)}) ORDER BY id`, uniq);
|
|
135
|
+
const res = await this.query(`SELECT id, scope, slug, ${jsonColumnText("pg", "frontmatter")}, body, rev, mtime_ms, size_bytes FROM ${PG_MEMORY_ENGINE_TABLES.entry} WHERE id IN (${inPlaceholders(1, uniq.length)}) ORDER BY id`, uniq);
|
|
118
136
|
return res.rows.map((r) => this.entryFromRow(r));
|
|
119
137
|
}
|
|
120
138
|
async search(query, scopes, opts) {
|
|
@@ -132,7 +150,7 @@ export class PgMemoryEngineBackend {
|
|
|
132
150
|
if (Array.isArray(qv) && qv.length === this.embedder.dimensions) {
|
|
133
151
|
const vecParam = scopes.length + 1;
|
|
134
152
|
const bandExpr = bands ? `(CASE WHEN ${originMarkedSqlPredicate("frontmatter")} THEN 1 ELSE 0 END) ASC, ` : "";
|
|
135
|
-
const res = await this.query(`SELECT id, scope, slug, frontmatter, rev, mtime_ms, size_bytes, embedding <=> $${vecParam}::vector AS dist
|
|
153
|
+
const res = await this.query(`SELECT id, scope, slug, ${jsonColumnText("pg", "frontmatter")}, rev, mtime_ms, size_bytes, embedding <=> $${vecParam}::vector AS dist
|
|
136
154
|
FROM ${PG_MEMORY_ENGINE_TABLES.entry}
|
|
137
155
|
WHERE scope IN (${inPlaceholders(1, scopes.length)}) AND embedding IS NOT NULL
|
|
138
156
|
ORDER BY ${bandExpr}embedding <=> $${vecParam}::vector ASC, id ASC
|
|
@@ -141,16 +159,16 @@ export class PgMemoryEngineBackend {
|
|
|
141
159
|
return hits.sort(contractOrder);
|
|
142
160
|
}
|
|
143
161
|
}
|
|
144
|
-
const res = await this.query(`SELECT id, scope, slug, frontmatter, body, rev, mtime_ms, size_bytes, embedding FROM ${PG_MEMORY_ENGINE_TABLES.entry} WHERE scope IN (${inPlaceholders(1, scopes.length)})`, [...scopes]);
|
|
162
|
+
const res = await this.query(`SELECT id, scope, slug, ${jsonColumnText("pg", "frontmatter")}, body, rev, mtime_ms, size_bytes, ${jsonColumnText("pg", "embedding")} FROM ${PG_MEMORY_ENGINE_TABLES.entry} WHERE scope IN (${inPlaceholders(1, scopes.length)})`, [...scopes]);
|
|
145
163
|
const queryVec = this.embedder ? await this.embedder.embed(query).catch(() => null) : null;
|
|
146
164
|
const dims = this.embedder?.dimensions;
|
|
147
165
|
const scored = [];
|
|
148
166
|
for (const raw of res.rows) {
|
|
149
167
|
const r = raw;
|
|
150
|
-
const fm =
|
|
168
|
+
const fm = readFrontmatter(r.frontmatter, r.id);
|
|
151
169
|
if (queryVec && Array.isArray(queryVec) && queryVec.length === dims && r.embedding != null) {
|
|
152
|
-
const stored =
|
|
153
|
-
if (
|
|
170
|
+
const stored = readEmbedding(r.embedding, r.id);
|
|
171
|
+
if (stored !== undefined && stored.length === dims) {
|
|
154
172
|
scored.push({ ...this.headerFromRow(r), score: cosineDistance(queryVec, stored) });
|
|
155
173
|
continue;
|
|
156
174
|
}
|
|
@@ -233,10 +251,10 @@ export class PgMemoryEngineBackend {
|
|
|
233
251
|
return;
|
|
234
252
|
}
|
|
235
253
|
const entry = rawEntry;
|
|
236
|
-
const prior = await this.query(`SELECT scope, slug, rev, frontmatter FROM ${T} WHERE id = $1`, [entry.id]);
|
|
254
|
+
const prior = await this.query(`SELECT scope, slug, rev, ${jsonColumnText("pg", "frontmatter")} FROM ${T} WHERE id = $1`, [entry.id]);
|
|
237
255
|
const priorRow = prior.rows.length > 0 ? prior.rows[0] : undefined;
|
|
238
256
|
{
|
|
239
|
-
const committedBaseFm = rememberBatchCommitted(originBaseline, entry.id, priorRow !== undefined ?
|
|
257
|
+
const committedBaseFm = rememberBatchCommitted(originBaseline, entry.id, priorRow !== undefined ? readFrontmatter(priorRow.frontmatter, entry.id) : undefined);
|
|
240
258
|
const refusal = originWhitewashRefusal("add", committedBaseFm, entry.frontmatter);
|
|
241
259
|
if (refusal !== undefined) {
|
|
242
260
|
report.conflicts.push({ op: "add", id: entry.id, reason: refusal });
|
|
@@ -304,7 +322,7 @@ export class PgMemoryEngineBackend {
|
|
|
304
322
|
slug = `${entry.slug}-${n}`;
|
|
305
323
|
}
|
|
306
324
|
}
|
|
307
|
-
const cur = await this.query(`SELECT slug, rev, scope, frontmatter FROM ${PG_MEMORY_ENGINE_TABLES.entry} WHERE id = $1`, [patch.id]);
|
|
325
|
+
const cur = await this.query(`SELECT slug, rev, scope, ${jsonColumnText("pg", "frontmatter")} FROM ${PG_MEMORY_ENGINE_TABLES.entry} WHERE id = $1`, [patch.id]);
|
|
308
326
|
if (cur.rows.length === 0) {
|
|
309
327
|
report.conflicts.push({ op: patch.op, id: patch.id, reason: "unknown id (entry not found in backend)" });
|
|
310
328
|
return;
|
|
@@ -312,7 +330,7 @@ export class PgMemoryEngineBackend {
|
|
|
312
330
|
const currentRev = String(cur.rows[0].rev);
|
|
313
331
|
const currentSlug = String(cur.rows[0].slug);
|
|
314
332
|
const currentScope = String(cur.rows[0].scope);
|
|
315
|
-
const committedFm =
|
|
333
|
+
const committedFm = readFrontmatter(cur.rows[0].frontmatter, patch.id);
|
|
316
334
|
const committedBaseFm = rememberBatchCommitted(originBaseline, patch.id, committedFm);
|
|
317
335
|
if (patch.op === "update" && patch.entry !== undefined) {
|
|
318
336
|
if (committedFm.provenance?.kind === "repo_file") {
|
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import { jaccardDistance, termSet } from "@sema-agent/core";
|
|
2
|
+
import { readFrontmatterColumn } from "./memory-engine-vector-util.js";
|
|
3
|
+
import { decodeJsonColumn, foldJsonColumn, jsonColumnText } from "./sql-json-column.js";
|
|
2
4
|
import { computeEntryRev, serializeEntryFile } from "@sema-agent/core";
|
|
3
5
|
import { pgHasUnstorable } from "./pg-safe-json.js";
|
|
4
6
|
import { assertMemoryScopeWidth, memoryEntryKeyWidthRefusal } from "./memory-key-guards.js";
|
|
@@ -41,8 +43,13 @@ function isPrimaryKeyDup(err) {
|
|
|
41
43
|
function inPlaceholders(count) {
|
|
42
44
|
return Array.from({ length: count }, () => "?").join(", ");
|
|
43
45
|
}
|
|
44
|
-
function
|
|
45
|
-
|
|
46
|
+
function readFrontmatter(cell, id) {
|
|
47
|
+
try {
|
|
48
|
+
return decodeJsonColumn(cell, readFrontmatterColumn, `${TIDB_MEMORY_ENGINE_TABLES.entry}.frontmatter for entry ${String(id)}`);
|
|
49
|
+
}
|
|
50
|
+
catch (e) {
|
|
51
|
+
throw foldJsonColumn(e, (m) => new Error(`memory-engine: ${m} — refusing rather than serve a note whose frontmatter (the origin/exposure law carrier) nobody can read`));
|
|
52
|
+
}
|
|
46
53
|
}
|
|
47
54
|
export class TiDBMemoryEngineBackend {
|
|
48
55
|
pool;
|
|
@@ -72,7 +79,7 @@ export class TiDBMemoryEngineBackend {
|
|
|
72
79
|
return `${fm.name ?? slug} ${fm.description ?? ""} ${body}`;
|
|
73
80
|
}
|
|
74
81
|
headerFromRow(r) {
|
|
75
|
-
const fm =
|
|
82
|
+
const fm = readFrontmatter(r.frontmatter, r.id);
|
|
76
83
|
return {
|
|
77
84
|
id: r.id,
|
|
78
85
|
slug: r.slug,
|
|
@@ -92,7 +99,7 @@ export class TiDBMemoryEngineBackend {
|
|
|
92
99
|
id: r.id,
|
|
93
100
|
slug: r.slug,
|
|
94
101
|
scope: r.scope,
|
|
95
|
-
frontmatter:
|
|
102
|
+
frontmatter: readFrontmatter(r.frontmatter, r.id),
|
|
96
103
|
body: r.body,
|
|
97
104
|
rev: r.rev,
|
|
98
105
|
};
|
|
@@ -101,14 +108,14 @@ export class TiDBMemoryEngineBackend {
|
|
|
101
108
|
const uniq = [...new Set(scopes)];
|
|
102
109
|
if (uniq.length === 0)
|
|
103
110
|
return [];
|
|
104
|
-
const rows = await this.selectRows(`SELECT id, scope, slug, frontmatter, rev, mtime_ms, size_bytes FROM ${TIDB_MEMORY_ENGINE_TABLES.entry} WHERE scope IN (${inPlaceholders(uniq.length)}) ORDER BY id`, uniq);
|
|
111
|
+
const rows = await this.selectRows(`SELECT id, scope, slug, ${jsonColumnText("tidb", "frontmatter")}, rev, mtime_ms, size_bytes FROM ${TIDB_MEMORY_ENGINE_TABLES.entry} WHERE scope IN (${inPlaceholders(uniq.length)}) ORDER BY id`, uniq);
|
|
105
112
|
return rows.map((r) => this.headerFromRow(r));
|
|
106
113
|
}
|
|
107
114
|
async getByIds(ids) {
|
|
108
115
|
const uniq = [...new Set(ids)];
|
|
109
116
|
if (uniq.length === 0)
|
|
110
117
|
return [];
|
|
111
|
-
const rows = await this.selectRows(`SELECT id, scope, slug, frontmatter, body, rev, mtime_ms, size_bytes FROM ${TIDB_MEMORY_ENGINE_TABLES.entry} WHERE id IN (${inPlaceholders(uniq.length)}) ORDER BY id`, uniq);
|
|
118
|
+
const rows = await this.selectRows(`SELECT id, scope, slug, ${jsonColumnText("tidb", "frontmatter")}, body, rev, mtime_ms, size_bytes FROM ${TIDB_MEMORY_ENGINE_TABLES.entry} WHERE id IN (${inPlaceholders(uniq.length)}) ORDER BY id`, uniq);
|
|
112
119
|
return rows.map((r) => this.entryFromRow(r));
|
|
113
120
|
}
|
|
114
121
|
async search(query, scopes, opts) {
|
|
@@ -121,11 +128,11 @@ export class TiDBMemoryEngineBackend {
|
|
|
121
128
|
if (q.size === 0)
|
|
122
129
|
return [];
|
|
123
130
|
const uniq = [...new Set(scopes)];
|
|
124
|
-
const rows = await this.selectRows(`SELECT id, scope, slug, frontmatter, body, rev, mtime_ms, size_bytes FROM ${TIDB_MEMORY_ENGINE_TABLES.entry} WHERE scope IN (${inPlaceholders(uniq.length)})`, uniq);
|
|
131
|
+
const rows = await this.selectRows(`SELECT id, scope, slug, ${jsonColumnText("tidb", "frontmatter")}, body, rev, mtime_ms, size_bytes FROM ${TIDB_MEMORY_ENGINE_TABLES.entry} WHERE scope IN (${inPlaceholders(uniq.length)})`, uniq);
|
|
125
132
|
const scored = [];
|
|
126
133
|
for (const raw of rows) {
|
|
127
134
|
const r = raw;
|
|
128
|
-
const fm =
|
|
135
|
+
const fm = readFrontmatter(r.frontmatter, r.id);
|
|
129
136
|
const d = jaccardDistance(q, this.haystackOf(r.slug, fm, r.body));
|
|
130
137
|
if (d === null)
|
|
131
138
|
continue;
|
|
@@ -203,11 +210,11 @@ export class TiDBMemoryEngineBackend {
|
|
|
203
210
|
report.conflicts.push({ op: "add", id: entry.id, reason: "unstorable_bytes (utf8mb4 cannot store NUL/lone surrogates without mangling; strip them at the source — the store never rewrites content)" });
|
|
204
211
|
return;
|
|
205
212
|
}
|
|
206
|
-
const prior = await this.selectRows(`SELECT scope, slug, rev, frontmatter FROM ${T} WHERE id = ?`, [entry.id]);
|
|
213
|
+
const prior = await this.selectRows(`SELECT scope, slug, rev, ${jsonColumnText("tidb", "frontmatter")} FROM ${T} WHERE id = ?`, [entry.id]);
|
|
207
214
|
const priorRow = prior.length > 0 ? prior[0] : undefined;
|
|
208
215
|
let existsById = priorRow !== undefined;
|
|
209
216
|
{
|
|
210
|
-
const committedBaseFm = rememberBatchCommitted(originBaseline, entry.id, priorRow !== undefined ?
|
|
217
|
+
const committedBaseFm = rememberBatchCommitted(originBaseline, entry.id, priorRow !== undefined ? readFrontmatter(priorRow.frontmatter, entry.id) : undefined);
|
|
211
218
|
const refusal = originWhitewashRefusal("add", committedBaseFm, entry.frontmatter);
|
|
212
219
|
if (refusal !== undefined) {
|
|
213
220
|
report.conflicts.push({ op: "add", id: entry.id, reason: refusal });
|
|
@@ -288,7 +295,7 @@ export class TiDBMemoryEngineBackend {
|
|
|
288
295
|
slug = `${entry.slug}-${n}`;
|
|
289
296
|
}
|
|
290
297
|
}
|
|
291
|
-
const cur = await this.selectRows(`SELECT slug, rev, scope, frontmatter FROM ${T} WHERE id = ?`, [patch.id]);
|
|
298
|
+
const cur = await this.selectRows(`SELECT slug, rev, scope, ${jsonColumnText("tidb", "frontmatter")} FROM ${T} WHERE id = ?`, [patch.id]);
|
|
292
299
|
if (cur.length === 0) {
|
|
293
300
|
report.conflicts.push({ op: patch.op, id: patch.id, reason: "unknown id (entry not found in backend)" });
|
|
294
301
|
return;
|
|
@@ -296,7 +303,7 @@ export class TiDBMemoryEngineBackend {
|
|
|
296
303
|
const currentRev = String(cur[0].rev);
|
|
297
304
|
const currentSlug = String(cur[0].slug);
|
|
298
305
|
const currentScope = String(cur[0].scope);
|
|
299
|
-
const committedFm =
|
|
306
|
+
const committedFm = readFrontmatter(cur[0].frontmatter, patch.id);
|
|
300
307
|
const committedBaseFm = rememberBatchCommitted(originBaseline, patch.id, committedFm);
|
|
301
308
|
if (patch.op === "update" && patch.entry !== undefined) {
|
|
302
309
|
if (committedFm.provenance?.kind === "repo_file") {
|
|
@@ -1,2 +1,28 @@
|
|
|
1
|
+
/** memory-engine 方言实现共用的纯工具函数。三个距离/词集函数(termSet/jaccardDistance/cosineDistance)
|
|
2
|
+
* 在 core(1.268.0 起公开导出,单一语义源)——消费者直引 core([ref] 兼容面全清:此处转口已删)。
|
|
3
|
+
* `isUniqueViolation` 重新内联:它当年随 core 的 pg **示例适配器**导出面走,core 1.424 按 clay 包卫生令
|
|
4
|
+
* 把整个 examples 面退出 npm 包([ref] BREAKING)⇒ re-export 断源。原为逐字抄自 core 1.423
|
|
5
|
+
* `examples/adapters/pg-adapter.js`(纯谓词,无 byte-equal 契约约束);[ref] P1-① 后**码面判据**
|
|
6
|
+
* 改走属主 `sql-errors.ts`,只留 message 补充臂(理由在函数体内)。 */
|
|
7
|
+
import type { MemoryEntryFrontmatter } from "@sema-agent/core";
|
|
8
|
+
import { type JsonColumnReader } from "./sql-json-column.js";
|
|
1
9
|
export declare function isUniqueViolation(err: unknown): boolean;
|
|
10
|
+
/**
|
|
11
|
+
* `agent_memory_engine_entry.frontmatter` 的**形读者**。
|
|
12
|
+
*
|
|
13
|
+
* 🔴 判据只到「一只 JSON 对象」为止:键表属主在 core(`MemoryEntryFrontmatter`),而 core 没有导出
|
|
14
|
+
* 验证器 —— 在这里手写一份等价 zod 就是设计稿明令禁止的第二份键集。这一条最小判据已经把坏体四形全关在
|
|
15
|
+
* 门外,而此前这里是**零判据**(裸 `JSON.parse` + `as MemoryEntryFrontmatter`)。
|
|
16
|
+
*
|
|
17
|
+
* 🔴 失败方向 = **拒**,而且这一列不是普通载荷:`frontmatter` 是 **origin/exposure 不可变律**的载体
|
|
18
|
+
* (`committedOriginOf` / `exposureBandOf` / `distilledCarriage` 全读它)。一行读不出形的 frontmatter 被
|
|
19
|
+
* 当成 `{}` 端出去 = 那条笔记的 external-origin 标记与曝光带**静默消失** —— 洗白检查(`originWhitewashRefusal`)
|
|
20
|
+
* 拿到的「已提交基线」于是是空的,一次真的洗白会被放过。
|
|
21
|
+
*/
|
|
22
|
+
export declare const readFrontmatterColumn: JsonColumnReader<MemoryEntryFrontmatter>;
|
|
23
|
+
/**
|
|
24
|
+
* `agent_memory_engine_entry.embedding` 的**形读者** = 本店既有的那一条判据(`Array.isArray`)。
|
|
25
|
+
* 维数判据仍在调用点(它要跟 embedder 的 `dimensions` 比,那是运行期事实,不是列的形)。
|
|
26
|
+
*/
|
|
27
|
+
export declare const readEmbeddingColumn: JsonColumnReader<unknown[]>;
|
|
2
28
|
//# sourceMappingURL=memory-engine-vector-util.d.ts.map
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { isPgUniqueViolation } from "./sql-errors.js";
|
|
2
|
+
import { createObjectColumnReader, readJsonArrayColumn } from "./sql-json-column.js";
|
|
2
3
|
export function isUniqueViolation(err) {
|
|
3
4
|
if (isPgUniqueViolation(err))
|
|
4
5
|
return true;
|
|
@@ -6,4 +7,6 @@ export function isUniqueViolation(err) {
|
|
|
6
7
|
return false;
|
|
7
8
|
return /duplicate key|unique constraint|already exists/i.test(String(err.message ?? ""));
|
|
8
9
|
}
|
|
10
|
+
export const readFrontmatterColumn = createObjectColumnReader();
|
|
11
|
+
export const readEmbeddingColumn = readJsonArrayColumn;
|
|
9
12
|
//# sourceMappingURL=memory-engine-vector-util.js.map
|
|
@@ -50,6 +50,12 @@ export interface MemorySyncStore {
|
|
|
50
50
|
export declare const MEMORY_SYNC_INDEXES: readonly IndexSpec[];
|
|
51
51
|
/** Idempotent DDL for the sync plane (PG dialect). Call once at startup. */
|
|
52
52
|
export declare function ensurePgMemorySyncSchema(query: PgQueryFn): Promise<void>;
|
|
53
|
+
/**
|
|
54
|
+
* 本店三只 JSON 承载列的**折码点**(唯一)—— 本店没有枚举式 corrupt 码,既有的「读不出来」口径就是
|
|
55
|
+
* 响亮抛(`claimLeaseMs` 守卫 / dup id 一律 fail-loud)⇒ 折成响亮裸 `Error`,点名 `<表>.<列>` + 行标识
|
|
56
|
+
* (S-550 车①② 裁定,沿用;不新铸码)。两个孪生共用本函数(MySQL 腿 import 它 —— 一条折码规则一个写者)。
|
|
57
|
+
*/
|
|
58
|
+
export declare function refusingMemorySyncRow<T>(read: () => T): T;
|
|
53
59
|
/** Options for {@link PgMemorySyncStore}. */
|
|
54
60
|
export interface PgMemorySyncStoreOptions {
|
|
55
61
|
/** How long a `claimDue` lease holds a row invisible before a crashed worker's claim self-heals.
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { ensureIndexes } from "./ensure-index.js";
|
|
2
2
|
import { pgSafeJsonStringify, pgProtocolJsonStringify } from "./pg-safe-json.js";
|
|
3
3
|
import { assertMemoryScopeWidth } from "./memory-key-guards.js";
|
|
4
|
+
import { decodeJsonColumn, decodeNullableJsonColumn, foldJsonColumn, jsonColumnText, readNonNullJsonValueColumn, readOpaqueJsonValueColumn, readStringRecordColumn, } from "./sql-json-column.js";
|
|
4
5
|
export const PG_MEMORY_SYNC_TABLES = {
|
|
5
6
|
syncCursor: "agent_memory_engine_sync_cursor",
|
|
6
7
|
pushQueue: "agent_memory_engine_push_queue",
|
|
@@ -28,10 +29,13 @@ export async function ensurePgMemorySyncSchema(query) {
|
|
|
28
29
|
)`);
|
|
29
30
|
await ensureIndexes(query, "pg", MEMORY_SYNC_INDEXES);
|
|
30
31
|
}
|
|
31
|
-
function
|
|
32
|
-
|
|
33
|
-
return
|
|
34
|
-
|
|
32
|
+
export function refusingMemorySyncRow(read) {
|
|
33
|
+
try {
|
|
34
|
+
return read();
|
|
35
|
+
}
|
|
36
|
+
catch (e) {
|
|
37
|
+
throw foldJsonColumn(e, (m) => new Error(`memory-sync store: ${m} — refusing rather than serve a sync row nobody can read`));
|
|
38
|
+
}
|
|
35
39
|
}
|
|
36
40
|
function toMs(v) {
|
|
37
41
|
return typeof v === "number" ? v : Number(v);
|
|
@@ -48,14 +52,14 @@ export class PgMemorySyncStore {
|
|
|
48
52
|
this.claimLeaseMs = lease;
|
|
49
53
|
}
|
|
50
54
|
async getCursor(scope, peer) {
|
|
51
|
-
const res = await this.query(`SELECT base_revs, updated_at_ms FROM ${PG_MEMORY_SYNC_TABLES.syncCursor} WHERE scope = $1 AND peer = $2`, [scope, peer]);
|
|
55
|
+
const res = await this.query(`SELECT ${jsonColumnText("pg", "base_revs")}, updated_at_ms FROM ${PG_MEMORY_SYNC_TABLES.syncCursor} WHERE scope = $1 AND peer = $2`, [scope, peer]);
|
|
52
56
|
const row = res.rows[0];
|
|
53
57
|
if (!row)
|
|
54
58
|
return undefined;
|
|
55
59
|
return {
|
|
56
60
|
scope,
|
|
57
61
|
peer,
|
|
58
|
-
baseRevs:
|
|
62
|
+
baseRevs: refusingMemorySyncRow(() => decodeJsonColumn(row.base_revs, readStringRecordColumn, `${PG_MEMORY_SYNC_TABLES.syncCursor}.base_revs for scope ${scope} peer ${peer}`)),
|
|
59
63
|
updatedAtMs: toMs(row.updated_at_ms),
|
|
60
64
|
};
|
|
61
65
|
}
|
|
@@ -83,12 +87,12 @@ export class PgMemorySyncStore {
|
|
|
83
87
|
)
|
|
84
88
|
UPDATE ${PG_MEMORY_SYNC_TABLES.pushQueue} q SET next_attempt_at_ms = $3
|
|
85
89
|
FROM due WHERE q.id = due.id
|
|
86
|
-
RETURNING q.id, q.scope, q.entry_id,
|
|
90
|
+
RETURNING q.id, q.scope, q.entry_id, ${jsonColumnText("pg", "payload")}, q.attempts, q.last_error`, [nowMs, limit, leasedUntilMs]);
|
|
87
91
|
return res.rows.map((r) => ({
|
|
88
92
|
id: String(r.id),
|
|
89
93
|
scope: String(r.scope),
|
|
90
94
|
entryId: String(r.entry_id),
|
|
91
|
-
payload:
|
|
95
|
+
payload: refusingMemorySyncRow(() => decodeJsonColumn(r.payload, readOpaqueJsonValueColumn, `${PG_MEMORY_SYNC_TABLES.pushQueue}.payload for item ${String(r.id)}`)),
|
|
92
96
|
attempts: toMs(r.attempts),
|
|
93
97
|
lastError: r.last_error == null ? null : String(r.last_error),
|
|
94
98
|
leasedUntilMs,
|
|
@@ -138,7 +142,7 @@ export class PgMemoryHistoryStore {
|
|
|
138
142
|
ON CONFLICT (id, rev) DO NOTHING`, params);
|
|
139
143
|
}
|
|
140
144
|
async historyOf(id) {
|
|
141
|
-
const res = await this.query(`SELECT id, rev, prev_rev, op, scope, snapshot, written_at_ms
|
|
145
|
+
const res = await this.query(`SELECT id, rev, prev_rev, op, scope, ${jsonColumnText("pg", "snapshot")}, written_at_ms
|
|
142
146
|
FROM ${PG_MEMORY_SYNC_TABLES.history}
|
|
143
147
|
WHERE id = $1
|
|
144
148
|
ORDER BY written_at_ms ASC, rev ASC`, [id]);
|
|
@@ -156,7 +160,7 @@ export function historyRowFrom(r) {
|
|
|
156
160
|
prevRev: r.prev_rev == null ? null : String(r.prev_rev),
|
|
157
161
|
op: String(r.op),
|
|
158
162
|
scope: String(r.scope),
|
|
159
|
-
snapshot: r.snapshot
|
|
163
|
+
snapshot: refusingMemorySyncRow(() => decodeNullableJsonColumn(r.snapshot, readNonNullJsonValueColumn, `${PG_MEMORY_SYNC_TABLES.history}.snapshot for entry ${String(r.id)} rev ${String(r.rev)}`)) ?? null,
|
|
160
164
|
writtenAtMs: toMs(r.written_at_ms),
|
|
161
165
|
};
|
|
162
166
|
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import { historyRowFrom } from "./memory-sync-store-pg.js";
|
|
1
|
+
import { historyRowFrom, refusingMemorySyncRow } from "./memory-sync-store-pg.js";
|
|
2
|
+
import { decodeJsonColumn, jsonColumnText, readOpaqueJsonValueColumn, readStringRecordColumn } from "./sql-json-column.js";
|
|
2
3
|
import { ensureIndexes } from "./ensure-index.js";
|
|
3
4
|
import { MEMORY_SYNC_INDEXES, MEMORY_HISTORY_INDEXES, PG_MEMORY_SYNC_TABLES } from "./memory-sync-store-pg.js";
|
|
4
5
|
import { assertMemoryScopeWidth } from "./memory-key-guards.js";
|
|
@@ -30,11 +31,6 @@ export async function ensureTiDBMemorySyncSchema(pool) {
|
|
|
30
31
|
) COLLATE utf8mb4_bin`);
|
|
31
32
|
await ensureIndexes((text, params) => pool.query(text, params), "mysql", MEMORY_SYNC_INDEXES);
|
|
32
33
|
}
|
|
33
|
-
function fromJson(v) {
|
|
34
|
-
if (typeof v === "string")
|
|
35
|
-
return JSON.parse(v);
|
|
36
|
-
return v;
|
|
37
|
-
}
|
|
38
34
|
function toMs(v) {
|
|
39
35
|
return typeof v === "number" ? v : Number(v);
|
|
40
36
|
}
|
|
@@ -50,14 +46,14 @@ export class TiDBMemorySyncStore {
|
|
|
50
46
|
this.claimLeaseMs = lease;
|
|
51
47
|
}
|
|
52
48
|
async getCursor(scope, peer) {
|
|
53
|
-
const [rows] = await this.pool.query(`SELECT base_revs, updated_at_ms FROM ${TIDB_MEMORY_SYNC_TABLES.syncCursor} WHERE scope = ? AND peer = ?`, [scope, peer]);
|
|
49
|
+
const [rows] = await this.pool.query(`SELECT ${jsonColumnText("tidb", "base_revs")}, updated_at_ms FROM ${TIDB_MEMORY_SYNC_TABLES.syncCursor} WHERE scope = ? AND peer = ?`, [scope, peer]);
|
|
54
50
|
const row = rows[0];
|
|
55
51
|
if (!row)
|
|
56
52
|
return undefined;
|
|
57
53
|
return {
|
|
58
54
|
scope,
|
|
59
55
|
peer,
|
|
60
|
-
baseRevs:
|
|
56
|
+
baseRevs: refusingMemorySyncRow(() => decodeJsonColumn(row.base_revs, readStringRecordColumn, `${TIDB_MEMORY_SYNC_TABLES.syncCursor}.base_revs for scope ${scope} peer ${peer}`)),
|
|
61
57
|
updatedAtMs: toMs(row.updated_at_ms),
|
|
62
58
|
};
|
|
63
59
|
}
|
|
@@ -76,7 +72,7 @@ export class TiDBMemorySyncStore {
|
|
|
76
72
|
if (limit <= 0)
|
|
77
73
|
return [];
|
|
78
74
|
const leasedUntilMs = nowMs + this.claimLeaseMs;
|
|
79
|
-
const [candidates] = await this.pool.query(`SELECT id, scope, entry_id, payload, attempts, next_attempt_at_ms, last_error
|
|
75
|
+
const [candidates] = await this.pool.query(`SELECT id, scope, entry_id, ${jsonColumnText("tidb", "payload")}, attempts, next_attempt_at_ms, last_error
|
|
80
76
|
FROM ${TIDB_MEMORY_SYNC_TABLES.pushQueue}
|
|
81
77
|
WHERE next_attempt_at_ms <= ?
|
|
82
78
|
ORDER BY next_attempt_at_ms, id
|
|
@@ -92,7 +88,7 @@ export class TiDBMemorySyncStore {
|
|
|
92
88
|
id: row.id,
|
|
93
89
|
scope: row.scope,
|
|
94
90
|
entryId: row.entry_id,
|
|
95
|
-
payload:
|
|
91
|
+
payload: refusingMemorySyncRow(() => decodeJsonColumn(row.payload, readOpaqueJsonValueColumn, `${TIDB_MEMORY_SYNC_TABLES.pushQueue}.payload for item ${row.id}`)),
|
|
96
92
|
attempts: toMs(row.attempts),
|
|
97
93
|
lastError: row.last_error == null ? null : String(row.last_error),
|
|
98
94
|
leasedUntilMs,
|
|
@@ -141,7 +137,7 @@ export class TiDBMemoryHistoryStore {
|
|
|
141
137
|
ON DUPLICATE KEY UPDATE id = id`, params);
|
|
142
138
|
}
|
|
143
139
|
async historyOf(id) {
|
|
144
|
-
const [rows] = await this.pool.query(`SELECT id, rev, prev_rev, op, scope, snapshot, written_at_ms
|
|
140
|
+
const [rows] = await this.pool.query(`SELECT id, rev, prev_rev, op, scope, ${jsonColumnText("tidb", "snapshot")}, written_at_ms
|
|
145
141
|
FROM ${TIDB_MEMORY_SYNC_TABLES.history}
|
|
146
142
|
WHERE id = ?
|
|
147
143
|
ORDER BY written_at_ms ASC, rev ASC`, [id]);
|