opcode-pg-memory 2.4.1 → 2.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/mcp/backfill-embeddings.d.ts +2 -1
- package/dist/src/mcp/backfill-embeddings.d.ts.map +1 -1
- package/dist/src/mcp/backfill-embeddings.js +49 -20
- package/dist/src/mcp/backfill-embeddings.js.map +1 -1
- package/dist/src/mcp/sync-health.d.ts +1 -1
- package/dist/src/mcp/sync-health.d.ts.map +1 -1
- package/dist/src/mcp/sync-health.js +2 -2
- package/dist/src/mcp/sync-health.js.map +1 -1
- package/dist/src/services/async-embedder.d.ts +2 -0
- package/dist/src/services/async-embedder.d.ts.map +1 -1
- package/dist/src/services/async-embedder.js +10 -0
- package/dist/src/services/async-embedder.js.map +1 -1
- package/package.json +1 -1
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { Pool } from 'pg';
|
|
2
2
|
export interface BackfillEmbeddingsInput {
|
|
3
|
-
/** Max observations to enqueue. 0 = unlimited. */
|
|
3
|
+
/** Max observations to enqueue. 0 = unlimited (all pending). */
|
|
4
4
|
limit?: number;
|
|
5
5
|
}
|
|
6
6
|
export interface BackfillEmbeddingsOutput {
|
|
@@ -11,6 +11,7 @@ export interface BackfillEmbeddingsOutput {
|
|
|
11
11
|
}
|
|
12
12
|
/**
|
|
13
13
|
* Enqueue observations with NULL embedding into the AsyncEmbedder queue.
|
|
14
|
+
* Uses cursor-based batching to process large backfills without loading everything at once.
|
|
14
15
|
* The embedder processes them sequentially (with Ollama cooldown/retry).
|
|
15
16
|
* Already-embedded observations are never touched (WHERE embedding IS NULL).
|
|
16
17
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"backfill-embeddings.d.ts","sourceRoot":"","sources":["../../../src/mcp/backfill-embeddings.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"backfill-embeddings.d.ts","sourceRoot":"","sources":["../../../src/mcp/backfill-embeddings.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAQ1B,MAAM,WAAW,uBAAuB;IACtC,gEAAgE;IAChE,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,wBAAwB;IACvC,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;CACd;AAED;;;;;GAKG;AACH,wBAAsB,kBAAkB,CACtC,KAAK,EAAE,uBAAuB,EAC9B,IAAI,EAAE,IAAI,GACT,OAAO,CAAC,wBAAwB,CAAC,CA+EnC"}
|
|
@@ -4,8 +4,10 @@ exports.backfillEmbeddings = backfillEmbeddings;
|
|
|
4
4
|
const async_embedder_1 = require("../services/async-embedder");
|
|
5
5
|
const logger_1 = require("../services/logger");
|
|
6
6
|
const logger = (0, logger_1.createLogger)('backfill-embeddings');
|
|
7
|
+
const BATCH_SIZE = 100;
|
|
7
8
|
/**
|
|
8
9
|
* Enqueue observations with NULL embedding into the AsyncEmbedder queue.
|
|
10
|
+
* Uses cursor-based batching to process large backfills without loading everything at once.
|
|
9
11
|
* The embedder processes them sequentially (with Ollama cooldown/retry).
|
|
10
12
|
* Already-embedded observations are never touched (WHERE embedding IS NULL).
|
|
11
13
|
*/
|
|
@@ -15,6 +17,9 @@ async function backfillEmbeddings(input, pool) {
|
|
|
15
17
|
return { enqueued: 0, skipped: 0, pending: 0, note: 'Embedder not initialized. Check plugin startup.' };
|
|
16
18
|
}
|
|
17
19
|
const limit = input.limit || 0;
|
|
20
|
+
let totalEnqueued = 0;
|
|
21
|
+
let totalSkipped = 0;
|
|
22
|
+
let cursor = null;
|
|
18
23
|
try {
|
|
19
24
|
// Count pending
|
|
20
25
|
const countResult = await pool.query(`SELECT count(*)::int AS cnt FROM observations WHERE importance >= 3 AND embedding IS NULL`);
|
|
@@ -22,33 +27,57 @@ async function backfillEmbeddings(input, pool) {
|
|
|
22
27
|
if (pending === 0) {
|
|
23
28
|
return { enqueued: 0, skipped: 0, pending: 0, note: 'All observations already have embeddings.' };
|
|
24
29
|
}
|
|
25
|
-
const
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
30
|
+
const maxToProcess = limit > 0 ? Math.min(limit, pending) : pending;
|
|
31
|
+
// Cursor-based batch loop
|
|
32
|
+
while (totalEnqueued < maxToProcess) {
|
|
33
|
+
const remaining = maxToProcess - totalEnqueued;
|
|
34
|
+
const batchSize = Math.min(BATCH_SIZE, remaining);
|
|
35
|
+
let query;
|
|
36
|
+
let params;
|
|
37
|
+
if (cursor) {
|
|
38
|
+
query = `SELECT id, tool_name, tool_input_summary, tool_output_summary, importance
|
|
39
|
+
FROM observations
|
|
40
|
+
WHERE importance >= 3 AND embedding IS NULL AND id > $1
|
|
41
|
+
ORDER BY id ASC
|
|
42
|
+
LIMIT $2`;
|
|
43
|
+
params = [cursor, batchSize];
|
|
44
|
+
}
|
|
45
|
+
else {
|
|
46
|
+
query = `SELECT id, tool_name, tool_input_summary, tool_output_summary, importance
|
|
47
|
+
FROM observations
|
|
48
|
+
WHERE importance >= 3 AND embedding IS NULL
|
|
49
|
+
ORDER BY id ASC
|
|
50
|
+
LIMIT $1`;
|
|
51
|
+
params = [batchSize];
|
|
52
|
+
}
|
|
53
|
+
const result = await pool.query(query, params);
|
|
54
|
+
if (result.rows.length === 0)
|
|
55
|
+
break;
|
|
56
|
+
for (const row of result.rows) {
|
|
57
|
+
if (totalEnqueued >= maxToProcess)
|
|
58
|
+
break;
|
|
59
|
+
const text = `[${row.tool_name || 'tool'}] ${row.tool_output_summary || row.tool_input_summary || ''}`;
|
|
60
|
+
if (!text || text.length <= 3) {
|
|
61
|
+
totalSkipped++;
|
|
62
|
+
continue;
|
|
63
|
+
}
|
|
64
|
+
embedder.enqueue('observations', row.id, text, row.importance);
|
|
65
|
+
totalEnqueued++;
|
|
66
|
+
cursor = row.id;
|
|
67
|
+
}
|
|
38
68
|
}
|
|
39
|
-
const
|
|
40
|
-
|
|
41
|
-
logger.info(`Backfill: enqueued ${enqueuedCount}, ${remaining} remaining`);
|
|
69
|
+
const remaining = pending - totalEnqueued;
|
|
70
|
+
logger.info(`Backfill: enqueued ${totalEnqueued}, skipped ${totalSkipped}, ${remaining} remaining`);
|
|
42
71
|
return {
|
|
43
|
-
enqueued:
|
|
44
|
-
skipped,
|
|
72
|
+
enqueued: totalEnqueued,
|
|
73
|
+
skipped: totalSkipped,
|
|
45
74
|
pending: remaining,
|
|
46
|
-
note: `Enqueued ${
|
|
75
|
+
note: `Enqueued ${totalEnqueued} observations (${remaining} remaining). Use sync_health() to monitor progress.`,
|
|
47
76
|
};
|
|
48
77
|
}
|
|
49
78
|
catch (err) {
|
|
50
79
|
logger.error('backfill_embeddings query failed', err);
|
|
51
|
-
return { enqueued:
|
|
80
|
+
return { enqueued: totalEnqueued, skipped: totalSkipped, pending: 0, note: 'Error: ' + err.message };
|
|
52
81
|
}
|
|
53
82
|
}
|
|
54
83
|
//# sourceMappingURL=backfill-embeddings.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"backfill-embeddings.js","sourceRoot":"","sources":["../../../src/mcp/backfill-embeddings.ts"],"names":[],"mappings":";;
|
|
1
|
+
{"version":3,"file":"backfill-embeddings.js","sourceRoot":"","sources":["../../../src/mcp/backfill-embeddings.ts"],"names":[],"mappings":";;AA0BA,gDAkFC;AA3GD,+DAA8D;AAC9D,+CAAkD;AAElD,MAAM,MAAM,GAAG,IAAA,qBAAY,EAAC,qBAAqB,CAAC,CAAC;AAEnD,MAAM,UAAU,GAAG,GAAG,CAAC;AAcvB;;;;;GAKG;AACI,KAAK,UAAU,kBAAkB,CACtC,KAA8B,EAC9B,IAAU;IAEV,MAAM,QAAQ,GAAG,IAAA,iCAAgB,GAAE,CAAC;IACpC,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,OAAO,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,iDAAiD,EAAE,CAAC;IAC1G,CAAC;IAED,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,IAAI,CAAC,CAAC;IAC/B,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,MAAM,GAAkB,IAAI,CAAC;IAEjC,IAAI,CAAC;QACH,gBAAgB;QAChB,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,KAAK,CAClC,2FAA2F,CAC5F,CAAC;QACF,MAAM,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC;QAE9C,IAAI,OAAO,KAAK,CAAC,EAAE,CAAC;YAClB,OAAO,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,2CAA2C,EAAE,CAAC;QACpG,CAAC;QAED,MAAM,YAAY,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;QAEpE,0BAA0B;QAC1B,OAAO,aAAa,GAAG,YAAY,EAAE,CAAC;YACpC,MAAM,SAAS,GAAG,YAAY,GAAG,aAAa,CAAC;YAC/C,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;YAElD,IAAI,KAAa,CAAC;YAClB,IAAI,MAAa,CAAC;YAElB,IAAI,MAAM,EAAE,CAAC;gBACX,KAAK,GAAG;;;;0BAIU,CAAC;gBACnB,MAAM,GAAG,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;YAC/B,CAAC;iBAAM,CAAC;gBACN,KAAK,GAAG;;;;0BAIU,CAAC;gBACnB,MAAM,GAAG,CAAC,SAAS,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;YAC/C,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,MAAM;YAEpC,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;gBAC9B,IAAI,aAAa,IAAI,YAAY;oBAAE,MAAM;gBAEzC,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,SAAS,IAAI,MAAM,KAAK,GAAG,CAAC,mBAAmB,IAAI,GAAG,CAAC,kBAAkB,IAAI,EAAE,EAAE,CAAC;gBACvG,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;oBAC9B,YAAY,EAAE,CAAC;oBACf,SAAS;gBACX,CAAC;gBACD,QAAQ,CAAC,OAAO,CAAC,cAAc,EAAE,GAAG,CAAC,EAAE,EAAE,IAAI,EAAE,GAAG,CAAC,UAAU,CAAC,CAAC;gBAC/D,aAAa,EAAE,CAAC;gBAChB,MAAM,GAAG,GAAG,CAAC,EAAE,CAAC;YAClB,CAAC;QACH,CAAC;QAED,MAAM,SAAS,GAAG,OAAO,GAAG,aAAa,CAAC;QAE1C,MAAM,CAAC,IAAI,CAAC,sBAAsB,aAAa,aAAa,YAAY,KAAK,SAAS,YAAY,CAAC,CAAC;QAEpG,OAAO;YACL,QAAQ,EAAE,aAAa;YACvB,OAAO,EAAE,YAAY;YACrB,OAAO,EAAE,SAAS;YAClB,IAAI,EAAE,YAAY,aAAa,kBAAkB,SAAS,qDAAqD;SAChH,CAAC;IACJ,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,MAAM,CAAC,KAAK,CAAC,kCAAkC,EAAE,GAAG,CAAC,CAAC;QACtD,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,OAAO,EAAE,YAAY,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,SAAS,GAAG,GAAG,CAAC,OAAO,EAAE,CAAC;IACvG,CAAC;AACH,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sync-health.d.ts","sourceRoot":"","sources":["../../../src/mcp/sync-health.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAU1B,MAAM,WAAW,eAAe;CAE/B;AAED,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,SAAS,GAAG,UAAU,GAAG,OAAO,CAAC;IACzC,YAAY,EAAE;QACZ,KAAK,EAAE,MAAM,CAAC;QACd,cAAc,EAAE,MAAM,CAAC;QACvB,aAAa,EAAE,MAAM,CAAC;QACtB,iBAAiB,EAAE,MAAM,CAAC;KAC3B,CAAC;IACF,QAAQ,EAAE;QACR,YAAY,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"sync-health.d.ts","sourceRoot":"","sources":["../../../src/mcp/sync-health.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAU1B,MAAM,WAAW,eAAe;CAE/B;AAED,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,SAAS,GAAG,UAAU,GAAG,OAAO,CAAC;IACzC,YAAY,EAAE;QACZ,KAAK,EAAE,MAAM,CAAC;QACd,cAAc,EAAE,MAAM,CAAC;QACvB,aAAa,EAAE,MAAM,CAAC;QACtB,iBAAiB,EAAE,MAAM,CAAC;KAC3B,CAAC;IACF,QAAQ,EAAE;QACR,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;QAC5B,oBAAoB,EAAE,MAAM,GAAG,IAAI,CAAC;KACrC,CAAC;IACF,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAMD;;;GAGG;AACH,wBAAsB,UAAU,CAC9B,MAAM,EAAE,eAAe,EACvB,IAAI,EAAE,IAAI,GACT,OAAO,CAAC,gBAAgB,CAAC,CAqE3B"}
|
|
@@ -36,13 +36,13 @@ async function syncHealth(_input, pool) {
|
|
|
36
36
|
return {
|
|
37
37
|
status: 'error',
|
|
38
38
|
observations: { total: 0, with_embedding: 0, embedding_pct: 0, sessions_with_obs: 0 },
|
|
39
|
-
embedder: { queue_length:
|
|
39
|
+
embedder: { queue_length: null, cooldown_remaining_s: null },
|
|
40
40
|
warnings: ['PG query failed: ' + err.message],
|
|
41
41
|
};
|
|
42
42
|
}
|
|
43
43
|
// Embedder stats
|
|
44
44
|
const embedder = (0, async_embedder_1.getAsyncEmbedder)();
|
|
45
|
-
const queueLength = embedder ? embedder.getQueueLength() :
|
|
45
|
+
const queueLength = embedder ? embedder.getQueueLength() : null;
|
|
46
46
|
const cooldownUntil = embedder ? embedder.getCooldownUntil() : null;
|
|
47
47
|
const cooldownRemaining = cooldownUntil
|
|
48
48
|
? Math.max(0, Math.round((cooldownUntil - Date.now()) / 1000))
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sync-health.js","sourceRoot":"","sources":["../../../src/mcp/sync-health.ts"],"names":[],"mappings":";;AAqCA,gCAwEC;AA5GD,+DAA8D;AAC9D,+CAAkD;AAElD,MAAM,MAAM,GAAG,IAAA,qBAAY,EAAC,aAAa,CAAC,CAAC;AAyB3C,+DAA+D;AAC/D,wBAAwB;AACxB,+DAA+D;AAE/D;;;GAGG;AACI,KAAK,UAAU,UAAU,CAC9B,MAAuB,EACvB,IAAU;IAEV,MAAM,QAAQ,GAAa,EAAE,CAAC;IAE9B,iBAAiB;IACjB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,eAAe,GAAG,CAAC,CAAC;IAExB,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC;;;;;KAKlC,CAAC,CAAC;QACH,KAAK,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC;QACtC,OAAO,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,QAAQ,IAAI,CAAC,CAAC;QAE3C,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC;;KAEtC,CAAC,CAAC;QACH,eAAe,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC;IACpD,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,MAAM,CAAC,KAAK,CAAC,6BAA6B,EAAE,GAAG,CAAC,CAAC;QACjD,OAAO;YACL,MAAM,EAAE,OAAO;YACf,YAAY,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,iBAAiB,EAAE,CAAC,EAAE;YACrF,QAAQ,EAAE,EAAE,YAAY,EAAE,
|
|
1
|
+
{"version":3,"file":"sync-health.js","sourceRoot":"","sources":["../../../src/mcp/sync-health.ts"],"names":[],"mappings":";;AAqCA,gCAwEC;AA5GD,+DAA8D;AAC9D,+CAAkD;AAElD,MAAM,MAAM,GAAG,IAAA,qBAAY,EAAC,aAAa,CAAC,CAAC;AAyB3C,+DAA+D;AAC/D,wBAAwB;AACxB,+DAA+D;AAE/D;;;GAGG;AACI,KAAK,UAAU,UAAU,CAC9B,MAAuB,EACvB,IAAU;IAEV,MAAM,QAAQ,GAAa,EAAE,CAAC;IAE9B,iBAAiB;IACjB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,eAAe,GAAG,CAAC,CAAC;IAExB,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC;;;;;KAKlC,CAAC,CAAC;QACH,KAAK,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC;QACtC,OAAO,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,QAAQ,IAAI,CAAC,CAAC;QAE3C,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC;;KAEtC,CAAC,CAAC;QACH,eAAe,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC;IACpD,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,MAAM,CAAC,KAAK,CAAC,6BAA6B,EAAE,GAAG,CAAC,CAAC;QACjD,OAAO;YACL,MAAM,EAAE,OAAO;YACf,YAAY,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,iBAAiB,EAAE,CAAC,EAAE;YACrF,QAAQ,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,oBAAoB,EAAE,IAAI,EAAE;YAC5D,QAAQ,EAAE,CAAC,mBAAmB,GAAG,GAAG,CAAC,OAAO,CAAC;SAC9C,CAAC;IACJ,CAAC;IAED,iBAAiB;IACjB,MAAM,QAAQ,GAAG,IAAA,iCAAgB,GAAE,CAAC;IACpC,MAAM,WAAW,GAAG,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;IAChE,MAAM,aAAa,GAAG,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;IACpE,MAAM,iBAAiB,GAAG,aAAa;QACrC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;QAC9D,CAAC,CAAC,IAAI,CAAC;IAET,WAAW;IACX,MAAM,YAAY,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,OAAO,GAAG,KAAK,CAAC,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACjF,IAAI,YAAY,GAAG,EAAE,EAAE,CAAC;QACtB,QAAQ,CAAC,IAAI,CACX,iCAAiC,KAAK,GAAG,OAAO,qDAAqD,CACtG,CAAC;IACJ,CAAC;IACD,IAAI,WAAW,GAAG,EAAE,EAAE,CAAC;QACrB,QAAQ,CAAC,IAAI,CACX,2BAA2B,WAAW,uCAAuC,CAC9E,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAA2B,QAAQ,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC;IAEtF,OAAO;QACL,MAAM;QACN,YAAY,EAAE;YACZ,KAAK;YACL,cAAc,EAAE,OAAO;YACvB,aAAa,EAAE,YAAY;YAC3B,iBAAiB,EAAE,eAAe;SACnC;QACD,QAAQ,EAAE;YACR,YAAY,EAAE,WAAW;YACzB,oBAAoB,EAAE,iBAAiB;SACxC;QACD,QAAQ;KACT,CAAC;AACJ,CAAC"}
|
|
@@ -6,6 +6,7 @@ export declare class AsyncEmbedder {
|
|
|
6
6
|
private cooldownUntil;
|
|
7
7
|
private readonly cooldownMs;
|
|
8
8
|
private readonly minImportance;
|
|
9
|
+
private droppedJobs;
|
|
9
10
|
constructor(pool: Pool, config?: {
|
|
10
11
|
cooldownMs?: number;
|
|
11
12
|
minImportance?: number;
|
|
@@ -16,6 +17,7 @@ export declare class AsyncEmbedder {
|
|
|
16
17
|
*/
|
|
17
18
|
enqueue(table: string, rowId: string, text: string, importance?: number): void;
|
|
18
19
|
getQueueLength(): number;
|
|
20
|
+
getDroppedCount(): number;
|
|
19
21
|
/**
|
|
20
22
|
* Returns timestamp (ms) when cooldown expires, or null if not cooling.
|
|
21
23
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"async-embedder.d.ts","sourceRoot":"","sources":["../../../src/services/async-embedder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"async-embedder.d.ts","sourceRoot":"","sources":["../../../src/services/async-embedder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAc1B,qBAAa,aAAa;IACxB,OAAO,CAAC,IAAI,CAAO;IACnB,OAAO,CAAC,KAAK,CAAsB;IACnC,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,aAAa,CAAK;IAC1B,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IACpC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAS;IACvC,OAAO,CAAC,WAAW,CAAK;gBAEZ,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,EAAE;QAAE,UAAU,CAAC,EAAE,MAAM,CAAC;QAAC,aAAa,CAAC,EAAE,MAAM,CAAA;KAAE;IAMhF;;;OAGG;IACH,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI;IAkB9E,cAAc,IAAI,MAAM;IAIxB,eAAe,IAAI,MAAM;IAIzB;;OAEG;IACH,gBAAgB,IAAI,MAAM,GAAG,IAAI;YAInB,KAAK;IAqDnB,OAAO,CAAC,KAAK;CAGd;AAQD,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,EAAE;IAAE,UAAU,CAAC,EAAE,MAAM,CAAC;IAAC,aAAa,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,aAAa,CAGrH;AAED,wBAAgB,gBAAgB,IAAI,aAAa,GAAG,IAAI,CAEvD"}
|
|
@@ -6,6 +6,7 @@ exports.getAsyncEmbedder = getAsyncEmbedder;
|
|
|
6
6
|
const embedding_1 = require("../utils/embedding");
|
|
7
7
|
const logger_1 = require("./logger");
|
|
8
8
|
const logger = (0, logger_1.createLogger)('async-embedder');
|
|
9
|
+
const MAX_QUEUE = 5000;
|
|
9
10
|
class AsyncEmbedder {
|
|
10
11
|
pool;
|
|
11
12
|
queue = [];
|
|
@@ -13,6 +14,7 @@ class AsyncEmbedder {
|
|
|
13
14
|
cooldownUntil = 0;
|
|
14
15
|
cooldownMs;
|
|
15
16
|
minImportance;
|
|
17
|
+
droppedJobs = 0;
|
|
16
18
|
constructor(pool, config) {
|
|
17
19
|
this.pool = pool;
|
|
18
20
|
this.cooldownMs = config?.cooldownMs ?? 300000;
|
|
@@ -27,6 +29,11 @@ class AsyncEmbedder {
|
|
|
27
29
|
return;
|
|
28
30
|
if (importance !== undefined && importance < this.minImportance)
|
|
29
31
|
return;
|
|
32
|
+
if (this.queue.length >= MAX_QUEUE) {
|
|
33
|
+
this.droppedJobs++;
|
|
34
|
+
logger.warn(`Embedder queue full (${MAX_QUEUE}), dropping observation ${rowId.substring(0, 8)}`);
|
|
35
|
+
return;
|
|
36
|
+
}
|
|
30
37
|
this.queue.push({ table, rowId, text });
|
|
31
38
|
if (!this.processing) {
|
|
32
39
|
this.processing = true;
|
|
@@ -36,6 +43,9 @@ class AsyncEmbedder {
|
|
|
36
43
|
getQueueLength() {
|
|
37
44
|
return this.queue.length;
|
|
38
45
|
}
|
|
46
|
+
getDroppedCount() {
|
|
47
|
+
return this.droppedJobs;
|
|
48
|
+
}
|
|
39
49
|
/**
|
|
40
50
|
* Returns timestamp (ms) when cooldown expires, or null if not cooling.
|
|
41
51
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"async-embedder.js","sourceRoot":"","sources":["../../../src/services/async-embedder.ts"],"names":[],"mappings":";;;
|
|
1
|
+
{"version":3,"file":"async-embedder.js","sourceRoot":"","sources":["../../../src/services/async-embedder.ts"],"names":[],"mappings":";;;AAkIA,8CAGC;AAED,4CAEC;AAxID,kDAAyD;AACzD,qCAAwC;AAExC,MAAM,MAAM,GAAG,IAAA,qBAAY,EAAC,gBAAgB,CAAC,CAAC;AAE9C,MAAM,SAAS,GAAG,IAAI,CAAC;AAQvB,MAAa,aAAa;IAChB,IAAI,CAAO;IACX,KAAK,GAAmB,EAAE,CAAC;IAC3B,UAAU,GAAG,KAAK,CAAC;IACnB,aAAa,GAAG,CAAC,CAAC;IACT,UAAU,CAAS;IACnB,aAAa,CAAS;IAC/B,WAAW,GAAG,CAAC,CAAC;IAExB,YAAY,IAAU,EAAE,MAAwD;QAC9E,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,UAAU,GAAG,MAAM,EAAE,UAAU,IAAI,MAAM,CAAC;QAC/C,IAAI,CAAC,aAAa,GAAG,MAAM,EAAE,aAAa,IAAI,CAAC,CAAC;IAClD,CAAC;IAED;;;OAGG;IACH,OAAO,CAAC,KAAa,EAAE,KAAa,EAAE,IAAY,EAAE,UAAmB;QACrE,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QACvC,IAAI,UAAU,KAAK,SAAS,IAAI,UAAU,GAAG,IAAI,CAAC,aAAa;YAAE,OAAO;QAExE,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;YACnC,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,MAAM,CAAC,IAAI,CAAC,wBAAwB,SAAS,2BAA2B,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;YACjG,OAAO;QACT,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAExC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;YACrB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;YACvB,YAAY,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;QACnC,CAAC;IACH,CAAC;IAED,cAAc;QACZ,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;IAC3B,CAAC;IAED,eAAe;QACb,OAAO,IAAI,CAAC,WAAW,CAAC;IAC1B,CAAC;IAED;;OAEG;IACH,gBAAgB;QACd,OAAO,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC;IACrE,CAAC;IAEO,KAAK,CAAC,KAAK;QACjB,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;YAC3B,MAAM,CAAC,IAAI,CAAC,2BAA2B,IAAI,CAAC,KAAK,CAAC,MAAM,uCAAuC,CAAC,CAAC;QACnG,CAAC;QAED,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7B,4DAA4D;YAC5D,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;gBACpC,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBAClD,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC;gBAC7C,SAAS;YACX,CAAC;YAED,gEAAgE;YAChE,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAE1B,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,IAAA,+BAAmB,GAAE,CAAC;gBACtC,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,uDAAuD;oBACvD,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;oBAChB,MAAM;gBACR,CAAC;gBAED,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBAC5D,IAAI,CAAC,SAAS,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBACzC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,yBAAyB;oBAC7C,SAAS;gBACX,CAAC;gBAED,6FAA6F;gBAC7F,4EAA4E;gBAC5E,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CACnB,UAAU,GAAG,CAAC,KAAK,iEAAiE,EACpF,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,CACvC,CAAC;gBACF,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,6BAA6B;YACnD,CAAC;YAAC,OAAO,GAAQ,EAAE,CAAC;gBAClB,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;gBAC9C,IAAI,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;oBAC3I,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC;oBAClD,MAAM,CAAC,IAAI,CAAC,oCAAoC,IAAI,CAAC,UAAU,GAAG,IAAI,GAAG,CAAC,CAAC;oBAC3E,6CAA6C;gBAC/C,CAAC;qBAAM,CAAC;oBACN,MAAM,CAAC,IAAI,CAAC,qBAAqB,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;oBAChD,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,kCAAkC;gBACxD,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;IAC1B,CAAC;IAEO,KAAK,CAAC,EAAU;QACtB,OAAO,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;IAC7C,CAAC;CACF;AA5GD,sCA4GC;AAED,+EAA+E;AAC/E,yBAAyB;AACzB,+EAA+E;AAE/E,IAAI,SAAS,GAAyB,IAAI,CAAC;AAE3C,SAAgB,iBAAiB,CAAC,IAAU,EAAE,MAAwD;IACpG,SAAS,GAAG,IAAI,aAAa,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAC5C,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAgB,gBAAgB;IAC9B,OAAO,SAAS,CAAC;AACnB,CAAC"}
|