@claude-flow/cli 3.53.0 → 3.54.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/helpers/helpers.manifest.json +4 -4
- package/.claude/helpers/hook-handler.cjs +88 -6
- package/.claude/helpers/intelligence.cjs +104 -2
- package/README.md +141 -44
- package/catalog-manifest.json +2 -2
- package/dist/src/blocking-stdio.d.ts +11 -0
- package/dist/src/blocking-stdio.js +23 -0
- package/dist/src/commands/memory.js +9 -1
- package/dist/src/commands/session.js +11 -37
- package/dist/src/index.js +3 -0
- package/dist/src/init/executor.js +1 -0
- package/dist/src/init/helpers-generator.js +65 -3
- package/dist/src/mcp-tools/agentdb-tools.js +13 -1
- package/dist/src/mcp-tools/hooks-tools.js +1 -1
- package/dist/src/mcp-tools/session-tools.js +4 -1
- package/dist/src/memory/embedding-q8.d.ts +19 -0
- package/dist/src/memory/embedding-q8.js +38 -0
- package/dist/src/memory/graph-edge-writer.js +3 -2
- package/dist/src/memory/lucene-bm25.js +11 -19
- package/dist/src/memory/memory-bridge.d.ts +8 -0
- package/dist/src/memory/memory-bridge.js +21 -2
- package/dist/src/memory/memory-initializer.d.ts +5 -0
- package/dist/src/memory/memory-initializer.js +12 -5
- package/dist/src/memory/shared-sqlite.d.ts +7 -0
- package/dist/src/memory/shared-sqlite.js +45 -0
- package/dist/src/mods/probe.d.ts +7 -0
- package/dist/src/mods/probe.js +29 -1
- package/dist/src/production/monitoring.d.ts +2 -0
- package/dist/src/production/monitoring.js +8 -3
- package/dist/src/production/rate-limiter.js +8 -5
- package/dist/src/production/retry.js +2 -0
- package/dist/src/services/git-workspace-identity.js +20 -3
- package/dist/src/services/worker-queue.d.ts +1 -0
- package/dist/src/services/worker-queue.js +9 -1
- package/package.json +2 -2
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* #3851: stdout/stderr writes to a pipe are asynchronous on POSIX, so the
|
|
3
|
+
* `process.exit()` calls that follow a command (bin wrappers, CLI.run error
|
|
4
|
+
* paths, commands that exit themselves) discard whatever the consumer has not
|
|
5
|
+
* read yet, i.e. everything past the pipe buffer (64 KiB). Switching the pipe
|
|
6
|
+
* handles to blocking mode makes every write complete before it returns, so
|
|
7
|
+
* every exit path is safe. Same technique as the MCP stdio server.
|
|
8
|
+
* Files and TTYs are already synchronous and are left alone.
|
|
9
|
+
*/
|
|
10
|
+
export function ensureBlockingStdio() {
|
|
11
|
+
for (const stream of [process.stdout, process.stderr]) {
|
|
12
|
+
const s = stream;
|
|
13
|
+
if (s.isTTY)
|
|
14
|
+
continue;
|
|
15
|
+
try {
|
|
16
|
+
s._handle?.setBlocking?.(true);
|
|
17
|
+
}
|
|
18
|
+
catch {
|
|
19
|
+
// best effort: never fail the CLI because stdio could not be switched
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
//# sourceMappingURL=blocking-stdio.js.map
|
|
@@ -9,6 +9,7 @@ import { distillCommand } from './memory-distill.js';
|
|
|
9
9
|
import { backupCommand } from './memory-backup.js';
|
|
10
10
|
import { countSiblingStoreRows } from '../memory/sibling-store.js';
|
|
11
11
|
import { resolveDbPath } from '../memory/memory-initializer.js';
|
|
12
|
+
import { MAX_LIST_EMBEDDINGS } from '../memory/embedding-q8.js';
|
|
12
13
|
import { existsSync } from 'node:fs';
|
|
13
14
|
import { siblingAgentDbPath } from '../memory/memory-bridge.js';
|
|
14
15
|
import { validateIdentifier } from '../mcp-tools/validate-input.js';
|
|
@@ -778,16 +779,23 @@ const listCommand = {
|
|
|
778
779
|
type: 'number',
|
|
779
780
|
default: 20
|
|
780
781
|
},
|
|
782
|
+
{
|
|
783
|
+
name: 'embeddings',
|
|
784
|
+
description: `ADR-472: with --format json, add each entry's stored embedding as int8+scale (embeddingQ8: {dims, scale, b64}); read-only, at most ${MAX_LIST_EMBEDDINGS} entries per call`,
|
|
785
|
+
type: 'boolean',
|
|
786
|
+
default: false
|
|
787
|
+
},
|
|
781
788
|
DB_PATH_OPTION
|
|
782
789
|
],
|
|
783
790
|
action: async (ctx) => {
|
|
784
791
|
const namespace = ctx.flags.namespace;
|
|
785
792
|
const limit = ctx.flags.limit;
|
|
793
|
+
const includeEmbedding = ctx.flags.embeddings === true && ctx.flags.format === 'json';
|
|
786
794
|
// Use sql.js directly for consistent data access
|
|
787
795
|
try {
|
|
788
796
|
const { listEntries, resolveDbPath: _rdbList } = await import('../memory/memory-initializer.js');
|
|
789
797
|
const dbPathList = _rdbList(ctx.flags.path);
|
|
790
|
-
const listResult = await listEntries({ namespace, limit, offset: 0, dbPath: dbPathList });
|
|
798
|
+
const listResult = await listEntries({ namespace, limit, offset: 0, dbPath: dbPathList, ...(includeEmbedding && { includeEmbedding: true }) });
|
|
791
799
|
if (!listResult.success) {
|
|
792
800
|
output.printError(`Failed to list: ${listResult.error}`);
|
|
793
801
|
return { success: false, exitCode: 1 };
|
|
@@ -7,6 +7,8 @@ import { confirm, input, select } from '../prompt.js';
|
|
|
7
7
|
import { callMCPTool, MCPClientError } from '../mcp-client.js';
|
|
8
8
|
import * as fs from 'fs';
|
|
9
9
|
import * as path from 'path';
|
|
10
|
+
import { stringify as stringifyYaml } from 'yaml';
|
|
11
|
+
import { gzipSync, gunzipSync } from 'node:zlib';
|
|
10
12
|
// Format date for display
|
|
11
13
|
function formatDate(dateStr) {
|
|
12
14
|
if (!dateStr)
|
|
@@ -561,7 +563,7 @@ const exportCommand = {
|
|
|
561
563
|
// Format output
|
|
562
564
|
let content;
|
|
563
565
|
if (exportFormat === 'yaml') {
|
|
564
|
-
content =
|
|
566
|
+
content = stringifyYaml(result.data);
|
|
565
567
|
}
|
|
566
568
|
else {
|
|
567
569
|
content = JSON.stringify(result.data, null, 2);
|
|
@@ -570,7 +572,8 @@ const exportCommand = {
|
|
|
570
572
|
const absolutePath = path.isAbsolute(outputPath)
|
|
571
573
|
? outputPath
|
|
572
574
|
: path.join(ctx.cwd, outputPath);
|
|
573
|
-
|
|
575
|
+
const bytes = compress ? gzipSync(content) : Buffer.from(content, 'utf-8');
|
|
576
|
+
fs.writeFileSync(absolutePath, bytes);
|
|
574
577
|
spinner.succeed('Session exported');
|
|
575
578
|
output.writeln();
|
|
576
579
|
// session_export returns the whole record under `data`; its stats live there.
|
|
@@ -587,14 +590,14 @@ const exportCommand = {
|
|
|
587
590
|
{ property: 'Agents', value: exportStats.agentCount ?? exportStats.agents ?? 0 },
|
|
588
591
|
{ property: 'Tasks', value: exportStats.taskCount ?? exportStats.tasks ?? 0 },
|
|
589
592
|
{ property: 'Memory Entries', value: exportStats.memoryEntries ?? 0 },
|
|
590
|
-
{ property: 'File Size', value: formatSize(
|
|
593
|
+
{ property: 'File Size', value: formatSize(bytes.length) }
|
|
591
594
|
]
|
|
592
595
|
});
|
|
593
596
|
output.writeln();
|
|
594
597
|
output.printSuccess(`Session exported to ${outputPath}`);
|
|
595
598
|
return {
|
|
596
599
|
success: true,
|
|
597
|
-
data: { sessionId, outputPath, format: exportFormat, size:
|
|
600
|
+
data: { sessionId, outputPath, format: exportFormat, size: bytes.length }
|
|
598
601
|
};
|
|
599
602
|
}
|
|
600
603
|
catch (error) {
|
|
@@ -647,9 +650,11 @@ const importCommand = {
|
|
|
647
650
|
try {
|
|
648
651
|
// The YAML written by `session export --format yaml` is display-only;
|
|
649
652
|
// there is no YAML parser here, so say so instead of failing obscurely.
|
|
650
|
-
if (
|
|
653
|
+
if (/\.ya?ml(?:\.gz)?$/i.test(absolutePath)) {
|
|
651
654
|
try {
|
|
652
|
-
|
|
655
|
+
const bytes = fs.readFileSync(absolutePath);
|
|
656
|
+
const content = bytes[0] === 0x1f && bytes[1] === 0x8b ? gunzipSync(bytes) : bytes;
|
|
657
|
+
JSON.parse(content.toString('utf-8'));
|
|
653
658
|
}
|
|
654
659
|
catch {
|
|
655
660
|
spinner.fail('Failed to import session');
|
|
@@ -783,37 +788,6 @@ function formatDuration(ms) {
|
|
|
783
788
|
}
|
|
784
789
|
return `${seconds}s`;
|
|
785
790
|
}
|
|
786
|
-
function toSimpleYaml(obj, indent = 0) {
|
|
787
|
-
// Simple YAML serializer (for basic types)
|
|
788
|
-
if (obj === null)
|
|
789
|
-
return 'null';
|
|
790
|
-
if (typeof obj === 'boolean')
|
|
791
|
-
return String(obj);
|
|
792
|
-
if (typeof obj === 'number')
|
|
793
|
-
return String(obj);
|
|
794
|
-
if (typeof obj === 'string')
|
|
795
|
-
return obj.includes(':') ? `"${obj}"` : obj;
|
|
796
|
-
const spaces = ' '.repeat(indent);
|
|
797
|
-
let result = '';
|
|
798
|
-
if (Array.isArray(obj)) {
|
|
799
|
-
for (const item of obj) {
|
|
800
|
-
result += `${spaces}- ${toSimpleYaml(item, indent + 1).trim()}\n`;
|
|
801
|
-
}
|
|
802
|
-
return result;
|
|
803
|
-
}
|
|
804
|
-
if (typeof obj === 'object') {
|
|
805
|
-
for (const [key, value] of Object.entries(obj)) {
|
|
806
|
-
if (typeof value === 'object' && value !== null) {
|
|
807
|
-
result += `${spaces}${key}:\n${toSimpleYaml(value, indent + 1)}`;
|
|
808
|
-
}
|
|
809
|
-
else {
|
|
810
|
-
result += `${spaces}${key}: ${toSimpleYaml(value, indent)}\n`;
|
|
811
|
-
}
|
|
812
|
-
}
|
|
813
|
-
return result;
|
|
814
|
-
}
|
|
815
|
-
return String(obj);
|
|
816
|
-
}
|
|
817
791
|
// Main session command
|
|
818
792
|
export const sessionCommand = {
|
|
819
793
|
name: 'session',
|
package/dist/src/index.js
CHANGED
|
@@ -19,6 +19,7 @@ import { output } from './output.js';
|
|
|
19
19
|
import { commands, getCommandsByCategory, getCommand, getCommandAsync, getCommandNames, getLazyCommandNames, hasCommand } from './commands/index.js';
|
|
20
20
|
import { suggestCommand } from './suggest.js';
|
|
21
21
|
import { runStartupUpdateCheck } from './update/index.js';
|
|
22
|
+
import { ensureBlockingStdio } from './blocking-stdio.js';
|
|
22
23
|
// Read version from package.json at runtime
|
|
23
24
|
function getPackageVersion() {
|
|
24
25
|
try {
|
|
@@ -66,6 +67,8 @@ export class CLI {
|
|
|
66
67
|
* Run the CLI with given arguments
|
|
67
68
|
*/
|
|
68
69
|
async run(args = process.argv.slice(2)) {
|
|
70
|
+
// #3851: keep piped output intact across the process.exit() calls below
|
|
71
|
+
ensureBlockingStdio();
|
|
69
72
|
try {
|
|
70
73
|
// #1791.2 — If the user invoked a lazy command (e.g. `hive-mind task`),
|
|
71
74
|
// pre-load it BEFORE parsing so the parser can build scoped flag
|
|
@@ -436,6 +436,7 @@ module.exports = commands;
|
|
|
436
436
|
// Literal-word root guard mirrored in the mod and both shipped classic helpers.
|
|
437
437
|
const ROOT_DELETE_CHECK_SOURCE = String.raw `function hasRootDelete(command, depth = 0) {
|
|
438
438
|
let word = '', quote = '', started = false, redirect = false
|
|
439
|
+
let ansiNul = false // a NUL inside $'...' ends that string, as in bash
|
|
439
440
|
let inRm = false, optionsEnded = false, recursive = false, force = false, root = false
|
|
440
441
|
const isRoot = (operand) => {
|
|
441
442
|
if (!operand.startsWith('/')) return false
|
|
@@ -447,22 +448,55 @@ const ROOT_DELETE_CHECK_SOURCE = String.raw `function hasRootDelete(command, dep
|
|
|
447
448
|
}
|
|
448
449
|
return parts.length === 0 || /[*?\[]/.test(parts[0])
|
|
449
450
|
}
|
|
451
|
+
// One backslash escape inside $'...', from the character after the backslash.
|
|
452
|
+
// Returns the text it stands for and the index of its last character. An unknown escape keeps its backslash.
|
|
453
|
+
// The command arrives lowercased, so an uppercase-U escape reads as a lowercase one: eight digits starting 0000 are one code point.
|
|
454
|
+
const ansiEscape = (s, at) => {
|
|
455
|
+
const c = s[at]
|
|
456
|
+
const simple = { a: '\x07', b: '\b', e: '\x1b', f: '\f', n: '\n', r: '\r', t: '\t', v: '\v', '\\': '\\', "'": "'", '"': '"', '?': '?' }
|
|
457
|
+
if (simple[c] !== undefined) return [simple[c], at]
|
|
458
|
+
const digits = (from, max, pattern) => pattern.exec(s.slice(from, from + max))?.[0]
|
|
459
|
+
const octal = digits(at, 3, /^[0-7]+/)
|
|
460
|
+
if (octal) return [String.fromCharCode(parseInt(octal, 8) & 255), at + octal.length - 1]
|
|
461
|
+
const hex = c === 'x' ? digits(at + 1, 2, /^[0-9a-f]+/) : undefined
|
|
462
|
+
if (hex) return [String.fromCharCode(parseInt(hex, 16)), at + hex.length]
|
|
463
|
+
const wide = c === 'u' ? (digits(at + 1, 8, /^0000[0-9a-f]{4}$/) ?? digits(at + 1, 4, /^[0-9a-f]+/)) : undefined
|
|
464
|
+
if (wide) return [String.fromCodePoint(parseInt(wide, 16)), at + wide.length]
|
|
465
|
+
if (c === 'c' && at + 1 < s.length) return [String.fromCharCode(s.charCodeAt(at + 1) & 31), at + 1]
|
|
466
|
+
return ['\\' + c, at]
|
|
467
|
+
}
|
|
450
468
|
const finishWord = () => {
|
|
451
469
|
if (!started) return false
|
|
452
470
|
// Literal shell strings (e.g. sh -c 'rm -rf /') also carried the old guard.
|
|
453
471
|
// Bound rescanning to four levels; beyond that retain its conservative check.
|
|
454
|
-
if (
|
|
472
|
+
if (/[\s;&|()\x60]/.test(word)) {
|
|
455
473
|
if (depth < 4 ? hasRootDelete(word, depth + 1) : word.includes('rm -rf /')) return true
|
|
456
474
|
}
|
|
457
475
|
if (!inRm) inRm = word === 'rm' || word.endsWith('/rm')
|
|
458
476
|
else if (!optionsEnded && word === '--') optionsEnded = true
|
|
459
477
|
else if (!optionsEnded && word.startsWith('-')) {
|
|
460
|
-
|
|
461
|
-
|
|
478
|
+
// GNU getopt accepts any unambiguous long-option prefix: --r, --recur, --forc.
|
|
479
|
+
recursive = recursive || (word.length > 2 && '--recursive'.startsWith(word)) || /^-[a-z]*r[a-z]*$/.test(word)
|
|
480
|
+
force = force || (word.length > 2 && '--force'.startsWith(word)) || /^-[a-z]*f[a-z]*$/.test(word)
|
|
462
481
|
} else root = root || isRoot(word)
|
|
463
482
|
word = ''; started = false
|
|
464
483
|
return inRm && recursive && force && root
|
|
465
484
|
}
|
|
485
|
+
// A command substitution's body, from just after its opening backtick. Bash removes a
|
|
486
|
+
// backslash only before $, a backtick or a backslash (and " inside double quotes) and
|
|
487
|
+
// drops backslash-newline; any other backslash stays for the nested scan.
|
|
488
|
+
const substitution = (from, inDouble) => {
|
|
489
|
+
let body = '', j = from
|
|
490
|
+
for (; j < command.length && command[j] !== '\x60'; j++) {
|
|
491
|
+
const next = command[j + 1]
|
|
492
|
+
if (command[j] === '\\' && j + 1 < command.length) {
|
|
493
|
+
if (next === '\n') { j++; continue }
|
|
494
|
+
if (next === '$' || next === '\x60' || next === '\\' || (inDouble && next === '"')) { body += next; j++; continue }
|
|
495
|
+
}
|
|
496
|
+
body += command[j]
|
|
497
|
+
}
|
|
498
|
+
return [j, depth < 4 ? hasRootDelete(body, depth + 1) : body.includes('rm -rf /')]
|
|
499
|
+
}
|
|
466
500
|
const finishCommand = () => {
|
|
467
501
|
const denied = inRm && recursive && force && root
|
|
468
502
|
inRm = optionsEnded = recursive = force = root = false
|
|
@@ -473,7 +507,25 @@ const ROOT_DELETE_CHECK_SOURCE = String.raw `function hasRootDelete(command, dep
|
|
|
473
507
|
const redirectionAmpersand = char === '&' && (redirect || command[i + 1] === '>')
|
|
474
508
|
redirect = false
|
|
475
509
|
if (quote) {
|
|
510
|
+
if (quote === "$'") {
|
|
511
|
+
if (char === "'") { quote = ''; ansiNul = false }
|
|
512
|
+
else if (char === '\\' && i + 1 < command.length) {
|
|
513
|
+
const [text, last] = ansiEscape(command, i + 1)
|
|
514
|
+
i = last
|
|
515
|
+
if (!ansiNul) { const nul = text.indexOf('\0'); if (nul < 0) word += text; else { word += text.slice(0, nul); ansiNul = true } }
|
|
516
|
+
} else if (!ansiNul) word += char
|
|
517
|
+
continue
|
|
518
|
+
}
|
|
476
519
|
if (char === quote) quote = ''
|
|
520
|
+
else if (quote === '"' && char === '\x60' && command.indexOf('\x60', i + 1) > i) {
|
|
521
|
+
const [end, denied] = substitution(i + 1, true)
|
|
522
|
+
if (denied) return true
|
|
523
|
+
// The substitution is gone from the word, as bash's empty output is (a quoted empty substitution glued to -rf leaves -rf).
|
|
524
|
+
// A # right after it is still inside the word: a placeholder keeps a rescan (sh -c "...")
|
|
525
|
+
// from reading it as the start of a comment.
|
|
526
|
+
if (command[end + 1] === '#') word += '\u0001'
|
|
527
|
+
i = end
|
|
528
|
+
}
|
|
477
529
|
else if (quote === '"' && char === '\\' && i + 1 < command.length &&
|
|
478
530
|
(command[i + 1] === '"' || command[i + 1] === '\\' || command[i + 1] === '$' ||
|
|
479
531
|
command.charCodeAt(i + 1) === 96 || command[i + 1] === '\n')) {
|
|
@@ -485,6 +537,16 @@ const ROOT_DELETE_CHECK_SOURCE = String.raw `function hasRootDelete(command, dep
|
|
|
485
537
|
if (char === '\\' && i + 1 < command.length) {
|
|
486
538
|
const next = command[++i]
|
|
487
539
|
if (next !== '\n') { word += next; started = true }
|
|
540
|
+
} else if (char === '\x60' && command.indexOf('\x60', i + 1) > i) {
|
|
541
|
+
// Command substitution: scan the body as its own command; the substitution
|
|
542
|
+
// stays inside the enclosing word, so the rm being parsed keeps its state,
|
|
543
|
+
// and the word has started (a # right after it is not a comment).
|
|
544
|
+
const [end, denied] = substitution(i + 1, false)
|
|
545
|
+
if (denied) return true
|
|
546
|
+
i = end; started = true
|
|
547
|
+
} else if (char === '$' && (command[i + 1] === "'" || command[i + 1] === '"')) {
|
|
548
|
+
// ANSI-C ($'...': escapes decoded) and locale ($"...") quoting: the $ is not part of the word.
|
|
549
|
+
quote = command[i + 1] === "'" ? "$'" : '"'; started = true; i++
|
|
488
550
|
} else if (char === '"' || char === "'") {
|
|
489
551
|
quote = char; started = true
|
|
490
552
|
} else if (char === '#' && !started) {
|
|
@@ -150,6 +150,10 @@ export const agentdbPatternStore = {
|
|
|
150
150
|
const confidence = validateScore(params.confidence, 0.8);
|
|
151
151
|
const bridge = await getBridge();
|
|
152
152
|
const result = await bridge.bridgeStorePattern({ pattern, type, confidence });
|
|
153
|
+
if (result && result.success === false) {
|
|
154
|
+
// #3691: the write was refused/not persisted — never acknowledge it.
|
|
155
|
+
return { success: false, error: `Pattern store failed: ${result.error ?? 'write not persisted'}`, controller: result.controller };
|
|
156
|
+
}
|
|
153
157
|
if (result) {
|
|
154
158
|
// #3288: `controller: 'reasoningBank'` is the ONLY label that means
|
|
155
159
|
// the healthy path ran. Every other label bridgeStorePattern can
|
|
@@ -176,12 +180,20 @@ export const agentdbPatternStore = {
|
|
|
176
180
|
const { storeEntry } = await import('../memory/memory-initializer.js');
|
|
177
181
|
const patternId = `pattern-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
|
178
182
|
const value = JSON.stringify({ pattern, type, confidence, _fallback: 'reasoningBank-unavailable' });
|
|
179
|
-
await storeEntry({
|
|
183
|
+
const stored = await storeEntry({
|
|
180
184
|
key: patternId,
|
|
181
185
|
value,
|
|
182
186
|
namespace: 'pattern',
|
|
183
187
|
tags: [type, 'reasoning-pattern', 'fallback'],
|
|
184
188
|
});
|
|
189
|
+
// #3691: storeEntry reports failure as {success:false}, not a throw.
|
|
190
|
+
if (!stored || stored.success !== true) {
|
|
191
|
+
return {
|
|
192
|
+
success: false,
|
|
193
|
+
error: `Pattern store failed: memory_store fallback did not persist (${stored?.error ?? 'no result'})`,
|
|
194
|
+
recommendation: 'Run agentdb_health to inspect controller registration and check that .swarm/memory.db is writable.',
|
|
195
|
+
};
|
|
196
|
+
}
|
|
185
197
|
return {
|
|
186
198
|
success: true,
|
|
187
199
|
// #3288: a caller must not have to already know to distrust a
|
|
@@ -3406,7 +3406,7 @@ export const hooksPatternStore = {
|
|
|
3406
3406
|
}
|
|
3407
3407
|
// Fallback: persist using memory-initializer store
|
|
3408
3408
|
let storeResult = { success: false };
|
|
3409
|
-
if (!reasoningResult) {
|
|
3409
|
+
if (!reasoningResult?.success) {
|
|
3410
3410
|
const storeFn = await getRealStoreFunction();
|
|
3411
3411
|
if (storeFn) {
|
|
3412
3412
|
try {
|
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
*/
|
|
6
6
|
import { existsSync, readFileSync, readdirSync, unlinkSync, statSync, writeFileSync } from 'node:fs';
|
|
7
7
|
import { randomUUID } from 'node:crypto';
|
|
8
|
+
import { gunzipSync } from 'node:zlib';
|
|
8
9
|
import { join } from 'node:path';
|
|
9
10
|
import { getProjectCwd } from './types.js';
|
|
10
11
|
import { mkdirRestricted, readFileMaybeEncrypted, writeFileRestricted, } from '../fs-secure.js';
|
|
@@ -683,7 +684,9 @@ export const sessionTools = [
|
|
|
683
684
|
if (!existsSync(inputPath))
|
|
684
685
|
return { error: `File not found: ${inputPath}` };
|
|
685
686
|
try {
|
|
686
|
-
|
|
687
|
+
const bytes = readFileSync(inputPath);
|
|
688
|
+
const content = bytes[0] === 0x1f && bytes[1] === 0x8b ? gunzipSync(bytes) : bytes;
|
|
689
|
+
parsed = JSON.parse(content.toString('utf-8'));
|
|
687
690
|
}
|
|
688
691
|
catch (e) {
|
|
689
692
|
return { error: `Invalid session JSON: ${e.message}` };
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ADR-472: a stored embedding as int8 + scale, for `memory list --embeddings`. Pure; no store access.
|
|
3
|
+
*/
|
|
4
|
+
/** Most embeddings `memory list --embeddings` will emit per call, and the longest vector it will encode. */
|
|
5
|
+
export declare const MAX_LIST_EMBEDDINGS = 500;
|
|
6
|
+
export declare const MAX_EMBEDDING_DIMS = 4096;
|
|
7
|
+
/** A stored embedding as int8 with one scale, base64: ~1/4 the bytes of rounded JSON floats, enough for a 2D projection or a cosine screen. */
|
|
8
|
+
export type EmbeddingQ8 = {
|
|
9
|
+
dims: number;
|
|
10
|
+
scale: number;
|
|
11
|
+
b64: string;
|
|
12
|
+
};
|
|
13
|
+
/**
|
|
14
|
+
* Read-only view of a stored embedding (a JSON array in the `embedding` column) as int8 + scale. Returns undefined for a missing,
|
|
15
|
+
* malformed, non-finite, constant-zero or over-long vector, so a caller never gets a guessed one. Each value is within scale/2 of the
|
|
16
|
+
* original (scale = max|v| / 127); cosine against the original stays above 0.9999 for unit-scale text embeddings.
|
|
17
|
+
*/
|
|
18
|
+
export declare function encodeEmbeddingQ8(stored: unknown): EmbeddingQ8 | undefined;
|
|
19
|
+
//# sourceMappingURL=embedding-q8.d.ts.map
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ADR-472: a stored embedding as int8 + scale, for `memory list --embeddings`. Pure; no store access.
|
|
3
|
+
*/
|
|
4
|
+
/** Most embeddings `memory list --embeddings` will emit per call, and the longest vector it will encode. */
|
|
5
|
+
export const MAX_LIST_EMBEDDINGS = 500;
|
|
6
|
+
export const MAX_EMBEDDING_DIMS = 4096;
|
|
7
|
+
/**
|
|
8
|
+
* Read-only view of a stored embedding (a JSON array in the `embedding` column) as int8 + scale. Returns undefined for a missing,
|
|
9
|
+
* malformed, non-finite, constant-zero or over-long vector, so a caller never gets a guessed one. Each value is within scale/2 of the
|
|
10
|
+
* original (scale = max|v| / 127); cosine against the original stays above 0.9999 for unit-scale text embeddings.
|
|
11
|
+
*/
|
|
12
|
+
export function encodeEmbeddingQ8(stored) {
|
|
13
|
+
if (stored === null || stored === undefined)
|
|
14
|
+
return undefined;
|
|
15
|
+
let values;
|
|
16
|
+
try {
|
|
17
|
+
values = typeof stored === 'string' ? JSON.parse(stored) : stored;
|
|
18
|
+
}
|
|
19
|
+
catch {
|
|
20
|
+
return undefined;
|
|
21
|
+
}
|
|
22
|
+
if (!Array.isArray(values) || values.length < 2 || values.length > MAX_EMBEDDING_DIMS)
|
|
23
|
+
return undefined;
|
|
24
|
+
let max = 0;
|
|
25
|
+
for (const v of values) {
|
|
26
|
+
if (typeof v !== 'number' || !Number.isFinite(v))
|
|
27
|
+
return undefined;
|
|
28
|
+
max = Math.max(max, Math.abs(v));
|
|
29
|
+
}
|
|
30
|
+
if (max === 0)
|
|
31
|
+
return undefined;
|
|
32
|
+
const scale = max / 127;
|
|
33
|
+
const bytes = new Int8Array(values.length);
|
|
34
|
+
for (let i = 0; i < values.length; i++)
|
|
35
|
+
bytes[i] = Math.max(-127, Math.min(127, Math.round(values[i] / scale)));
|
|
36
|
+
return { dims: values.length, scale: Number(scale.toPrecision(6)), b64: Buffer.from(bytes.buffer).toString('base64') };
|
|
37
|
+
}
|
|
38
|
+
//# sourceMappingURL=embedding-q8.js.map
|
|
@@ -34,6 +34,7 @@ import * as fs from 'fs';
|
|
|
34
34
|
import * as crypto from 'crypto';
|
|
35
35
|
import { getMemoryRoot } from './memory-initializer.js';
|
|
36
36
|
import { encodeEmbedding } from './embedding-quantization.js';
|
|
37
|
+
import { loadBetterSqlite3 } from './shared-sqlite.js';
|
|
37
38
|
// ============================================================================
|
|
38
39
|
// Lazy-cached better-sqlite3 db handle
|
|
39
40
|
// ============================================================================
|
|
@@ -152,8 +153,8 @@ export async function getBridgeDb(customDbPath, opts) {
|
|
|
152
153
|
// the try/catch below.
|
|
153
154
|
let BetterSqlite3;
|
|
154
155
|
try {
|
|
155
|
-
|
|
156
|
-
BetterSqlite3 =
|
|
156
|
+
// #3693: share AgentDB's better-sqlite3 so the idle close cannot detach its handle.
|
|
157
|
+
BetterSqlite3 = await loadBetterSqlite3();
|
|
157
158
|
}
|
|
158
159
|
catch {
|
|
159
160
|
return null;
|
|
@@ -159,34 +159,26 @@ function step3(w) {
|
|
|
159
159
|
}
|
|
160
160
|
return w;
|
|
161
161
|
}
|
|
162
|
+
// Longest suffixes first: Porter step 4 selects the longest matching suffix
|
|
163
|
+
// ONCE and applies its condition; a failed condition must not fall through to
|
|
164
|
+
// a shorter suffix (e.g. "argument": "ment" fails m>1, so "ent" is not tried).
|
|
162
165
|
const STEP4_SUFFIXES = [
|
|
163
|
-
'
|
|
164
|
-
'
|
|
166
|
+
'ement', 'ance', 'ence', 'able', 'ible', 'ment', 'ant', 'ent', 'ism',
|
|
167
|
+
'ate', 'iti', 'ous', 'ive', 'ize', 'ion', 'al', 'er', 'ic', 'ou',
|
|
165
168
|
];
|
|
166
169
|
function step4(w) {
|
|
167
170
|
for (const suf of STEP4_SUFFIXES) {
|
|
168
|
-
if (w.endsWith(suf))
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
if (suf === 'ion') {
|
|
172
|
-
const last = stem[stem.length - 1];
|
|
173
|
-
if (last === 's' || last === 't')
|
|
174
|
-
return stem;
|
|
175
|
-
}
|
|
176
|
-
else {
|
|
177
|
-
return stem;
|
|
178
|
-
}
|
|
179
|
-
}
|
|
180
|
-
}
|
|
181
|
-
}
|
|
182
|
-
// ION special-case
|
|
183
|
-
if (w.endsWith('ion')) {
|
|
184
|
-
const stem = w.slice(0, -3);
|
|
171
|
+
if (!w.endsWith(suf))
|
|
172
|
+
continue;
|
|
173
|
+
const stem = w.slice(0, w.length - suf.length);
|
|
185
174
|
if (measure(stem) > 1) {
|
|
175
|
+
if (suf !== 'ion')
|
|
176
|
+
return stem;
|
|
186
177
|
const last = stem[stem.length - 1];
|
|
187
178
|
if (last === 's' || last === 't')
|
|
188
179
|
return stem;
|
|
189
180
|
}
|
|
181
|
+
return w;
|
|
190
182
|
}
|
|
191
183
|
return w;
|
|
192
184
|
}
|
|
@@ -136,6 +136,8 @@ export declare function bridgeListEntries(options: {
|
|
|
136
136
|
includeContent?: boolean;
|
|
137
137
|
/** ADR-323: restrict rows to these provenance types. */
|
|
138
138
|
provenanceFilter?: string[];
|
|
139
|
+
/** ADR-472: include each entry's embedding as int8+scale (`embeddingQ8`); at most MAX_LIST_EMBEDDINGS rows. */
|
|
140
|
+
includeEmbedding?: boolean;
|
|
139
141
|
}): Promise<{
|
|
140
142
|
success: boolean;
|
|
141
143
|
entries: {
|
|
@@ -150,6 +152,11 @@ export declare function bridgeListEntries(options: {
|
|
|
150
152
|
/** #2073: Present when `includeContent: true` was requested. */
|
|
151
153
|
content?: string;
|
|
152
154
|
provenanceType?: string;
|
|
155
|
+
embeddingQ8?: {
|
|
156
|
+
dims: number;
|
|
157
|
+
scale: number;
|
|
158
|
+
b64: string;
|
|
159
|
+
};
|
|
153
160
|
}[];
|
|
154
161
|
total: number;
|
|
155
162
|
error?: string;
|
|
@@ -346,6 +353,7 @@ export declare function bridgeStorePattern(options: {
|
|
|
346
353
|
controller: string;
|
|
347
354
|
hasEmbedding?: boolean;
|
|
348
355
|
embeddingError?: string;
|
|
356
|
+
error?: string;
|
|
349
357
|
} | null>;
|
|
350
358
|
/**
|
|
351
359
|
* Search patterns via ReasoningBank controller.
|
|
@@ -17,6 +17,8 @@
|
|
|
17
17
|
* @module v3/cli/memory-bridge
|
|
18
18
|
*/
|
|
19
19
|
import { liveMemoryRowSql } from './live-memory-row.js';
|
|
20
|
+
import { resolveAgentdbBetterSqlite3 } from './shared-sqlite.js';
|
|
21
|
+
import { encodeEmbeddingQ8, MAX_LIST_EMBEDDINGS } from './embedding-q8.js';
|
|
20
22
|
import * as path from 'path';
|
|
21
23
|
import * as crypto from 'crypto';
|
|
22
24
|
import { realpathSync } from 'node:fs';
|
|
@@ -449,7 +451,8 @@ async function getRegistry(dbPath) {
|
|
|
449
451
|
try {
|
|
450
452
|
const attestationFile = path.join(adbDir, 'dist/src/security/AttestationLog.js');
|
|
451
453
|
if (fs.existsSync(attestationFile)) {
|
|
452
|
-
|
|
454
|
+
// #3693: same better-sqlite3 copy AgentDB uses, so closing this handle cannot disturb its WAL.
|
|
455
|
+
const Database = (resolveAgentdbBetterSqlite3() ?? cjsRequire('better-sqlite3'));
|
|
453
456
|
const swarmDir = path.resolve(process.cwd(), '.swarm');
|
|
454
457
|
if (!fs.existsSync(swarmDir))
|
|
455
458
|
fs.mkdirSync(swarmDir, { recursive: true });
|
|
@@ -1430,7 +1433,7 @@ export async function bridgeListEntries(options) {
|
|
|
1430
1433
|
ORDER BY updated_at DESC
|
|
1431
1434
|
LIMIT ? OFFSET ?
|
|
1432
1435
|
`);
|
|
1433
|
-
const rows = stmt.all(...filterParams, limit, offset);
|
|
1436
|
+
const rows = stmt.all(...filterParams, options.includeEmbedding ? Math.min(limit, MAX_LIST_EMBEDDINGS) : limit, offset);
|
|
1434
1437
|
for (const row of rows) {
|
|
1435
1438
|
const entry = {
|
|
1436
1439
|
// #2073: don't truncate id when content is requested — callers
|
|
@@ -1448,6 +1451,11 @@ export async function bridgeListEntries(options) {
|
|
|
1448
1451
|
if (options.includeContent) {
|
|
1449
1452
|
entry.content = row.content || '';
|
|
1450
1453
|
}
|
|
1454
|
+
if (options.includeEmbedding) {
|
|
1455
|
+
const q8 = encodeEmbeddingQ8(row.embedding);
|
|
1456
|
+
if (q8)
|
|
1457
|
+
entry.embeddingQ8 = q8;
|
|
1458
|
+
}
|
|
1451
1459
|
entries.push(entry);
|
|
1452
1460
|
}
|
|
1453
1461
|
}
|
|
@@ -2105,6 +2113,17 @@ export async function bridgeStorePattern(options) {
|
|
|
2105
2113
|
});
|
|
2106
2114
|
if (!result)
|
|
2107
2115
|
return null;
|
|
2116
|
+
// #3691: bridgeStoreEntry reports data-level failures (MutationGuard
|
|
2117
|
+
// rejection, nothing written) as a truthy {success:false}. Do not turn
|
|
2118
|
+
// that into a success receipt — refuse at this boundary.
|
|
2119
|
+
if (!result.success) {
|
|
2120
|
+
return {
|
|
2121
|
+
success: false,
|
|
2122
|
+
patternId: '',
|
|
2123
|
+
controller: 'bridge-fallback',
|
|
2124
|
+
error: result.error ?? 'pattern write was not persisted',
|
|
2125
|
+
};
|
|
2126
|
+
}
|
|
2108
2127
|
// Add to HNSW index for fast semantic search (bridgeStoreEntry stores SQL only)
|
|
2109
2128
|
if (result.rawEmbedding) {
|
|
2110
2129
|
try {
|
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
*
|
|
9
9
|
* @module v3/cli/memory-initializer
|
|
10
10
|
*/
|
|
11
|
+
import { type EmbeddingQ8 } from './embedding-q8.js';
|
|
11
12
|
/**
|
|
12
13
|
* ADR-323 — typed memory provenance. Distinguishes WHO/WHAT wrote a memory
|
|
13
14
|
* entry (a user's stated claim vs an agent's own output vs a tool result vs
|
|
@@ -504,6 +505,8 @@ export declare function listEntries(options: {
|
|
|
504
505
|
includeContent?: boolean;
|
|
505
506
|
/** ADR-323: restrict rows to these provenance types. */
|
|
506
507
|
provenanceFilter?: string[];
|
|
508
|
+
/** ADR-472: include each entry's embedding as int8+scale (`embeddingQ8`); at most MAX_LIST_EMBEDDINGS rows. Read-only. */
|
|
509
|
+
includeEmbedding?: boolean;
|
|
507
510
|
}): Promise<{
|
|
508
511
|
success: boolean;
|
|
509
512
|
entries: {
|
|
@@ -518,6 +521,8 @@ export declare function listEntries(options: {
|
|
|
518
521
|
/** #2073: Present when `includeContent: true` was requested. */
|
|
519
522
|
content?: string;
|
|
520
523
|
provenanceType?: string;
|
|
524
|
+
/** ADR-472: present when `includeEmbedding: true` and the row has a valid stored vector. */
|
|
525
|
+
embeddingQ8?: EmbeddingQ8;
|
|
521
526
|
}[];
|
|
522
527
|
total: number;
|
|
523
528
|
error?: string;
|
|
@@ -8,7 +8,9 @@
|
|
|
8
8
|
*
|
|
9
9
|
* @module v3/cli/memory-initializer
|
|
10
10
|
*/
|
|
11
|
+
import { loadBetterSqlite3 } from './shared-sqlite.js';
|
|
11
12
|
import { liveMemoryRowSql } from './live-memory-row.js';
|
|
13
|
+
import { encodeEmbeddingQ8, MAX_LIST_EMBEDDINGS } from './embedding-q8.js';
|
|
12
14
|
import * as fs from 'fs';
|
|
13
15
|
import * as path from 'path';
|
|
14
16
|
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
@@ -1407,8 +1409,8 @@ export async function recoverMemoryDatabase(dbPath, opts = {}) {
|
|
|
1407
1409
|
try {
|
|
1408
1410
|
// Module name behind a variable so TS does not statically resolve the
|
|
1409
1411
|
// optional native dep's types at build time (CI may not install them).
|
|
1410
|
-
|
|
1411
|
-
Database =
|
|
1412
|
+
// #3693: same better-sqlite3 identity as AgentDB (see shared-sqlite.ts).
|
|
1413
|
+
Database = await loadBetterSqlite3();
|
|
1412
1414
|
}
|
|
1413
1415
|
catch {
|
|
1414
1416
|
return await restoreFromBackup('no-native');
|
|
@@ -1556,8 +1558,8 @@ export async function repairVectorIndexes(dbPath, opts = {}) {
|
|
|
1556
1558
|
try {
|
|
1557
1559
|
// Module name behind a variable so TS does not statically resolve the
|
|
1558
1560
|
// optional native dep's types at build time (CI may not install them).
|
|
1559
|
-
|
|
1560
|
-
Database =
|
|
1561
|
+
// #3693: same better-sqlite3 identity as AgentDB (see shared-sqlite.ts).
|
|
1562
|
+
Database = await loadBetterSqlite3();
|
|
1561
1563
|
}
|
|
1562
1564
|
catch {
|
|
1563
1565
|
// Native module absent (e.g. WASM-only host). Statusline fix still covers
|
|
@@ -3018,7 +3020,7 @@ export async function listEntries(options) {
|
|
|
3018
3020
|
const countResult = countRows.length > 0 ? [{ values: countRows }] : [];
|
|
3019
3021
|
const total = countResult[0]?.values?.[0]?.[0] || 0;
|
|
3020
3022
|
// Get entries
|
|
3021
|
-
const safeLimit = parseInt(String(limit), 10) || 100;
|
|
3023
|
+
const safeLimit = Math.min(parseInt(String(limit), 10) || 100, options.includeEmbedding ? MAX_LIST_EMBEDDINGS : Number.MAX_SAFE_INTEGER);
|
|
3022
3024
|
const safeOffset = parseInt(String(offset), 10) || 0;
|
|
3023
3025
|
// #2120 — same NULL-as-active acceptance as the count above.
|
|
3024
3026
|
const listStmt = db.prepare(`SELECT id, key, namespace, content, embedding, access_count, created_at, updated_at, provenance_type
|
|
@@ -3051,6 +3053,11 @@ export async function listEntries(options) {
|
|
|
3051
3053
|
if (options.includeContent) {
|
|
3052
3054
|
entry.content = content || '';
|
|
3053
3055
|
}
|
|
3056
|
+
if (options.includeEmbedding) {
|
|
3057
|
+
const q8 = encodeEmbeddingQ8(embedding);
|
|
3058
|
+
if (q8)
|
|
3059
|
+
entry.embeddingQ8 = q8;
|
|
3060
|
+
}
|
|
3054
3061
|
entries.push(entry);
|
|
3055
3062
|
}
|
|
3056
3063
|
}
|