@wipcomputer/memory-crystal 0.7.30 → 0.7.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +43 -0
- package/SKILL.md +1 -1
- package/package.json +1 -1
- package/.worktrees/memory-crystal-private--cc-mini-fix-home-fallback/.env.example +0 -20
- package/.worktrees/memory-crystal-private--cc-mini-fix-home-fallback/.publish-skill.json +0 -1
- package/.worktrees/memory-crystal-private--cc-mini-fix-home-fallback/CHANGELOG.md +0 -1297
- package/.worktrees/memory-crystal-private--cc-mini-fix-home-fallback/CLA.md +0 -19
- package/.worktrees/memory-crystal-private--cc-mini-fix-home-fallback/LICENSE +0 -52
- package/.worktrees/memory-crystal-private--cc-mini-fix-home-fallback/README-ENTERPRISE.md +0 -226
- package/.worktrees/memory-crystal-private--cc-mini-fix-home-fallback/README.md +0 -151
- package/.worktrees/memory-crystal-private--cc-mini-fix-home-fallback/RELAY.md +0 -199
- package/.worktrees/memory-crystal-private--cc-mini-fix-home-fallback/SKILL.md +0 -462
- package/.worktrees/memory-crystal-private--cc-mini-fix-home-fallback/TECHNICAL.md +0 -656
- package/.worktrees/memory-crystal-private--cc-mini-fix-home-fallback/_trash/RELEASE-NOTES-v0-7-23.md +0 -48
- package/.worktrees/memory-crystal-private--cc-mini-fix-home-fallback/_trash/RELEASE-NOTES-v0-7-25.md +0 -24
- package/.worktrees/memory-crystal-private--cc-mini-fix-home-fallback/_trash/RELEASE-NOTES-v0-7-26.md +0 -7
- package/.worktrees/memory-crystal-private--cc-mini-fix-home-fallback/_trash/RELEASE-NOTES-v0-7-28.md +0 -31
- package/.worktrees/memory-crystal-private--cc-mini-fix-home-fallback/_trash/RELEASE-NOTES-v0-7-29.md +0 -28
- package/.worktrees/memory-crystal-private--cc-mini-fix-home-fallback/_trash/RELEASE-NOTES-v0-7-4.md +0 -64
- package/.worktrees/memory-crystal-private--cc-mini-fix-home-fallback/_trash/RELEASE-NOTES-v0-7-5.md +0 -19
- package/.worktrees/memory-crystal-private--cc-mini-fix-home-fallback/cloud/README.md +0 -116
- package/.worktrees/memory-crystal-private--cc-mini-fix-home-fallback/cloud/docs/gpt-system-instructions.md +0 -69
- package/.worktrees/memory-crystal-private--cc-mini-fix-home-fallback/cloud/migrations/0001_init.sql +0 -52
- package/.worktrees/memory-crystal-private--cc-mini-fix-home-fallback/migrations/0001_init.sql +0 -51
- package/.worktrees/memory-crystal-private--cc-mini-fix-home-fallback/migrations/0002_cloud_storage.sql +0 -49
- package/.worktrees/memory-crystal-private--cc-mini-fix-home-fallback/openclaw.plugin.json +0 -11
- package/.worktrees/memory-crystal-private--cc-mini-fix-home-fallback/package-lock.json +0 -4169
- package/.worktrees/memory-crystal-private--cc-mini-fix-home-fallback/package.json +0 -61
- package/.worktrees/memory-crystal-private--cc-mini-fix-home-fallback/scripts/crystal-capture.sh +0 -29
- package/.worktrees/memory-crystal-private--cc-mini-fix-home-fallback/scripts/deploy-cloud.sh +0 -153
- package/.worktrees/memory-crystal-private--cc-mini-fix-home-fallback/scripts/ldm-backup.sh +0 -116
- package/.worktrees/memory-crystal-private--cc-mini-fix-home-fallback/scripts/migrate-lance-to-sqlite.mjs +0 -218
- package/.worktrees/memory-crystal-private--cc-mini-fix-home-fallback/skills/memory/SKILL.md +0 -438
- package/.worktrees/memory-crystal-private--cc-mini-fix-home-fallback/wrangler-demo.toml +0 -8
- package/.worktrees/memory-crystal-private--cc-mini-fix-home-fallback/wrangler-mcp.toml +0 -24
- package/.worktrees/memory-crystal-private--cc-mini-release-notes-v0.7.30/.env.example +0 -20
- package/.worktrees/memory-crystal-private--cc-mini-release-notes-v0.7.30/.publish-skill.json +0 -1
- package/.worktrees/memory-crystal-private--cc-mini-release-notes-v0.7.30/CHANGELOG.md +0 -1297
- package/.worktrees/memory-crystal-private--cc-mini-release-notes-v0.7.30/CLA.md +0 -19
- package/.worktrees/memory-crystal-private--cc-mini-release-notes-v0.7.30/LICENSE +0 -52
- package/.worktrees/memory-crystal-private--cc-mini-release-notes-v0.7.30/README-ENTERPRISE.md +0 -226
- package/.worktrees/memory-crystal-private--cc-mini-release-notes-v0.7.30/README.md +0 -151
- package/.worktrees/memory-crystal-private--cc-mini-release-notes-v0.7.30/RELAY.md +0 -199
- package/.worktrees/memory-crystal-private--cc-mini-release-notes-v0.7.30/RELEASE-NOTES-v0.7.30.md +0 -29
- package/.worktrees/memory-crystal-private--cc-mini-release-notes-v0.7.30/SKILL.md +0 -462
- package/.worktrees/memory-crystal-private--cc-mini-release-notes-v0.7.30/TECHNICAL.md +0 -656
- package/.worktrees/memory-crystal-private--cc-mini-release-notes-v0.7.30/_trash/RELEASE-NOTES-v0-7-23.md +0 -48
- package/.worktrees/memory-crystal-private--cc-mini-release-notes-v0.7.30/_trash/RELEASE-NOTES-v0-7-25.md +0 -24
- package/.worktrees/memory-crystal-private--cc-mini-release-notes-v0.7.30/_trash/RELEASE-NOTES-v0-7-26.md +0 -7
- package/.worktrees/memory-crystal-private--cc-mini-release-notes-v0.7.30/_trash/RELEASE-NOTES-v0-7-28.md +0 -31
- package/.worktrees/memory-crystal-private--cc-mini-release-notes-v0.7.30/_trash/RELEASE-NOTES-v0-7-29.md +0 -28
- package/.worktrees/memory-crystal-private--cc-mini-release-notes-v0.7.30/_trash/RELEASE-NOTES-v0-7-4.md +0 -64
- package/.worktrees/memory-crystal-private--cc-mini-release-notes-v0.7.30/_trash/RELEASE-NOTES-v0-7-5.md +0 -19
- package/.worktrees/memory-crystal-private--cc-mini-release-notes-v0.7.30/cloud/README.md +0 -116
- package/.worktrees/memory-crystal-private--cc-mini-release-notes-v0.7.30/cloud/docs/gpt-system-instructions.md +0 -69
- package/.worktrees/memory-crystal-private--cc-mini-release-notes-v0.7.30/cloud/migrations/0001_init.sql +0 -52
- package/.worktrees/memory-crystal-private--cc-mini-release-notes-v0.7.30/migrations/0001_init.sql +0 -51
- package/.worktrees/memory-crystal-private--cc-mini-release-notes-v0.7.30/migrations/0002_cloud_storage.sql +0 -49
- package/.worktrees/memory-crystal-private--cc-mini-release-notes-v0.7.30/openclaw.plugin.json +0 -11
- package/.worktrees/memory-crystal-private--cc-mini-release-notes-v0.7.30/package-lock.json +0 -4169
- package/.worktrees/memory-crystal-private--cc-mini-release-notes-v0.7.30/package.json +0 -61
- package/.worktrees/memory-crystal-private--cc-mini-release-notes-v0.7.30/scripts/crystal-capture.sh +0 -29
- package/.worktrees/memory-crystal-private--cc-mini-release-notes-v0.7.30/scripts/deploy-cloud.sh +0 -153
- package/.worktrees/memory-crystal-private--cc-mini-release-notes-v0.7.30/scripts/ldm-backup.sh +0 -116
- package/.worktrees/memory-crystal-private--cc-mini-release-notes-v0.7.30/scripts/migrate-lance-to-sqlite.mjs +0 -218
- package/.worktrees/memory-crystal-private--cc-mini-release-notes-v0.7.30/skills/memory/SKILL.md +0 -438
- package/.worktrees/memory-crystal-private--cc-mini-release-notes-v0.7.30/wrangler-demo.toml +0 -8
- package/.worktrees/memory-crystal-private--cc-mini-release-notes-v0.7.30/wrangler-mcp.toml +0 -24
- package/_trash/RELEASE-NOTES-v0-7-23.md +0 -48
- package/_trash/RELEASE-NOTES-v0-7-25.md +0 -24
- package/_trash/RELEASE-NOTES-v0-7-26.md +0 -7
- package/_trash/RELEASE-NOTES-v0-7-28.md +0 -31
- package/_trash/RELEASE-NOTES-v0-7-29.md +0 -28
- package/_trash/RELEASE-NOTES-v0-7-4.md +0 -64
- package/_trash/RELEASE-NOTES-v0-7-5.md +0 -19
- package/_trash/RELEASE-NOTES-v0.7.30.md +0 -29
- package/dist/bridge.d.ts +0 -7
- package/dist/bridge.js +0 -14
- package/dist/bulk-copy.d.ts +0 -17
- package/dist/bulk-copy.js +0 -90
- package/dist/cc-hook.d.ts +0 -8
- package/dist/cc-hook.js +0 -368
- package/dist/cc-poller.d.ts +0 -1
- package/dist/cc-poller.js +0 -550
- package/dist/chunk-25LXQJ4Z.js +0 -110
- package/dist/chunk-2DRXIRQW.js +0 -97
- package/dist/chunk-2ZNH5F6E.js +0 -1281
- package/dist/chunk-3G3SFYYI.js +0 -288
- package/dist/chunk-3RG5ZIWI.js +0 -10
- package/dist/chunk-3S6TI23B.js +0 -97
- package/dist/chunk-3VFIJYS4.js +0 -818
- package/dist/chunk-52QE3YI3.js +0 -1169
- package/dist/chunk-57RP3DIN.js +0 -1205
- package/dist/chunk-5HSZ4W2P.js +0 -62
- package/dist/chunk-645IPXW3.js +0 -290
- package/dist/chunk-7A7ELD4C.js +0 -1205
- package/dist/chunk-7FYY4GZM.js +0 -1205
- package/dist/chunk-7IUE7ODU.js +0 -254
- package/dist/chunk-7RMLKZIS.js +0 -108
- package/dist/chunk-AA3OPP4Z.js +0 -432
- package/dist/chunk-AEWLSYPH.js +0 -72
- package/dist/chunk-ASSZDR6I.js +0 -108
- package/dist/chunk-AYRJVWUC.js +0 -1205
- package/dist/chunk-CCYI5O3D.js +0 -148
- package/dist/chunk-D3I3ZSE2.js +0 -411
- package/dist/chunk-DACSKLY6.js +0 -219
- package/dist/chunk-DW5B4BL7.js +0 -108
- package/dist/chunk-EKSACBTJ.js +0 -1070
- package/dist/chunk-EXEZZADG.js +0 -248
- package/dist/chunk-F3Y7EL7K.js +0 -83
- package/dist/chunk-FBQWSDPC.js +0 -1328
- package/dist/chunk-FHRZNOMW.js +0 -1205
- package/dist/chunk-IM7N24MT.js +0 -129
- package/dist/chunk-IPNYIXFK.js +0 -1178
- package/dist/chunk-J7MRSZIO.js +0 -167
- package/dist/chunk-JITKI2OI.js +0 -106
- package/dist/chunk-JWZXYVET.js +0 -1068
- package/dist/chunk-KCQUXVYT.js +0 -108
- package/dist/chunk-KOQ43OX6.js +0 -1281
- package/dist/chunk-KYVWO6ZM.js +0 -1069
- package/dist/chunk-L3VHARQH.js +0 -413
- package/dist/chunk-LBWDS6BE.js +0 -288
- package/dist/chunk-LOVAHSQV.js +0 -411
- package/dist/chunk-LQOYCAGG.js +0 -446
- package/dist/chunk-LWAIPJ2W.js +0 -146
- package/dist/chunk-M5DHKW7M.js +0 -127
- package/dist/chunk-MBKCIJHM.js +0 -1328
- package/dist/chunk-MK42FMEG.js +0 -147
- package/dist/chunk-MOBMYHKL.js +0 -1205
- package/dist/chunk-MPLTNMRG.js +0 -67
- package/dist/chunk-NIJCVN3O.js +0 -147
- package/dist/chunk-NZCFSZQ7.js +0 -1205
- package/dist/chunk-O2UITJGH.js +0 -465
- package/dist/chunk-OCRA44AZ.js +0 -108
- package/dist/chunk-P3KJR66H.js +0 -117
- package/dist/chunk-PEK6JH65.js +0 -432
- package/dist/chunk-PJ6FFKEX.js +0 -77
- package/dist/chunk-PLUBBZYR.js +0 -800
- package/dist/chunk-PNKVD2UK.js +0 -26
- package/dist/chunk-PSQZURHO.js +0 -229
- package/dist/chunk-SGL6ISBJ.js +0 -1061
- package/dist/chunk-SJABZZT5.js +0 -97
- package/dist/chunk-TD3P3K32.js +0 -1199
- package/dist/chunk-TMDZJJKV.js +0 -288
- package/dist/chunk-UNHVZB5G.js +0 -411
- package/dist/chunk-VAFTWSTE.js +0 -1061
- package/dist/chunk-VNFXFQBB.js +0 -217
- package/dist/chunk-X3GVFKSJ.js +0 -1205
- package/dist/chunk-XZ3S56RQ.js +0 -1061
- package/dist/chunk-Y72C7F6O.js +0 -148
- package/dist/chunk-YLICP577.js +0 -1205
- package/dist/chunk-YX6AXLVK.js +0 -159
- package/dist/chunk-ZCQYHTNU.js +0 -146
- package/dist/cli.d.ts +0 -1
- package/dist/cli.js +0 -1105
- package/dist/cloud-crystal.js +0 -6
- package/dist/core.d.ts +0 -232
- package/dist/core.js +0 -12
- package/dist/crypto.d.ts +0 -20
- package/dist/crypto.js +0 -27
- package/dist/crystal-capture.sh +0 -29
- package/dist/crystal-serve.d.ts +0 -4
- package/dist/crystal-serve.js +0 -252
- package/dist/dev-update-SZ2Z4WCQ.js +0 -6
- package/dist/discover.d.ts +0 -30
- package/dist/discover.js +0 -177
- package/dist/doctor.d.ts +0 -9
- package/dist/doctor.js +0 -334
- package/dist/dream-weaver.d.ts +0 -8
- package/dist/dream-weaver.js +0 -56
- package/dist/file-sync.d.ts +0 -48
- package/dist/file-sync.js +0 -18
- package/dist/installer.d.ts +0 -61
- package/dist/installer.js +0 -676
- package/dist/ldm-backup.sh +0 -116
- package/dist/ldm.d.ts +0 -50
- package/dist/ldm.js +0 -32
- package/dist/mcp-server.d.ts +0 -1
- package/dist/mcp-server.js +0 -265
- package/dist/migrate.d.ts +0 -1
- package/dist/migrate.js +0 -89
- package/dist/mirror-sync.d.ts +0 -1
- package/dist/mirror-sync.js +0 -159
- package/dist/oc-backfill.d.ts +0 -19
- package/dist/oc-backfill.js +0 -74
- package/dist/openclaw.d.ts +0 -5
- package/dist/openclaw.js +0 -423
- package/dist/pair.d.ts +0 -4
- package/dist/pair.js +0 -75
- package/dist/poller.d.ts +0 -1
- package/dist/poller.js +0 -634
- package/dist/role.d.ts +0 -24
- package/dist/role.js +0 -13
- package/dist/search-pipeline-4K4OJSSS.js +0 -255
- package/dist/search-pipeline-4PRS6LI7.js +0 -280
- package/dist/search-pipeline-7UJMXPLO.js +0 -280
- package/dist/search-pipeline-DQTRLGBH.js +0 -74
- package/dist/search-pipeline-HNG37REH.js +0 -282
- package/dist/search-pipeline-IZFPLBUB.js +0 -280
- package/dist/search-pipeline-MID6F26Q.js +0 -73
- package/dist/search-pipeline-N52JZFNN.js +0 -282
- package/dist/search-pipeline-OPB2PRQQ.js +0 -280
- package/dist/search-pipeline-VXTE5HAD.js +0 -262
- package/dist/search-pipeline-XHFKADRG.js +0 -73
- package/dist/staging.d.ts +0 -29
- package/dist/staging.js +0 -21
- package/dist/summarize.d.ts +0 -19
- package/dist/summarize.js +0 -10
- package/dist/worker-demo.js +0 -186
- package/dist/worker-mcp.js +0 -404
- package/dist/worker.js +0 -137
package/dist/chunk-TMDZJJKV.js
DELETED
|
@@ -1,288 +0,0 @@
|
|
|
1
|
-
import {
|
|
2
|
-
decrypt,
|
|
3
|
-
decryptJSON,
|
|
4
|
-
encrypt,
|
|
5
|
-
encryptJSON,
|
|
6
|
-
loadRelayKey
|
|
7
|
-
} from "./chunk-7RMLKZIS.js";
|
|
8
|
-
import {
|
|
9
|
-
ldmPaths,
|
|
10
|
-
resolveStatePath,
|
|
11
|
-
stateWritePath
|
|
12
|
-
} from "./chunk-DACSKLY6.js";
|
|
13
|
-
|
|
14
|
-
// src/file-sync.ts
|
|
15
|
-
import { createHash } from "crypto";
|
|
16
|
-
import {
|
|
17
|
-
existsSync,
|
|
18
|
-
mkdirSync,
|
|
19
|
-
readFileSync,
|
|
20
|
-
writeFileSync,
|
|
21
|
-
unlinkSync,
|
|
22
|
-
readdirSync,
|
|
23
|
-
statSync
|
|
24
|
-
} from "fs";
|
|
25
|
-
import { join, relative, dirname } from "path";
|
|
26
|
-
var RELAY_URL = process.env.CRYSTAL_RELAY_URL || "";
|
|
27
|
-
var RELAY_TOKEN = process.env.CRYSTAL_RELAY_TOKEN || "";
|
|
28
|
-
var EXCLUDE_PATTERNS = [
|
|
29
|
-
"memory/crystal.db",
|
|
30
|
-
// DB syncs via delta chunks, not file copy
|
|
31
|
-
"memory/crystal.db-wal",
|
|
32
|
-
"memory/crystal.db-shm",
|
|
33
|
-
"memory/crystal.db.bak",
|
|
34
|
-
"memory/crystal.db.tmp",
|
|
35
|
-
"memory/lance/",
|
|
36
|
-
// LanceDB (deprecated, not synced)
|
|
37
|
-
"state/",
|
|
38
|
-
// Local state files (watermarks, etc.)
|
|
39
|
-
"secrets/",
|
|
40
|
-
// Encryption keys, tokens
|
|
41
|
-
"staging/",
|
|
42
|
-
// Staging pipeline (Core-only)
|
|
43
|
-
"bin/",
|
|
44
|
-
// Local scripts
|
|
45
|
-
".DS_Store"
|
|
46
|
-
];
|
|
47
|
-
function shouldExclude(relativePath) {
|
|
48
|
-
for (const pattern of EXCLUDE_PATTERNS) {
|
|
49
|
-
if (relativePath === pattern || relativePath.startsWith(pattern)) return true;
|
|
50
|
-
}
|
|
51
|
-
const parts = relativePath.split("/");
|
|
52
|
-
for (const part of parts) {
|
|
53
|
-
if (part.startsWith(".") && part !== ".ldm") return true;
|
|
54
|
-
}
|
|
55
|
-
return false;
|
|
56
|
-
}
|
|
57
|
-
function scanDir(baseDir, currentDir, entries) {
|
|
58
|
-
if (!existsSync(currentDir)) return;
|
|
59
|
-
const items = readdirSync(currentDir);
|
|
60
|
-
for (const item of items) {
|
|
61
|
-
const fullPath = join(currentDir, item);
|
|
62
|
-
const relPath = relative(baseDir, fullPath);
|
|
63
|
-
if (shouldExclude(relPath)) continue;
|
|
64
|
-
let stat;
|
|
65
|
-
try {
|
|
66
|
-
stat = statSync(fullPath);
|
|
67
|
-
} catch {
|
|
68
|
-
continue;
|
|
69
|
-
}
|
|
70
|
-
if (stat.isDirectory()) {
|
|
71
|
-
scanDir(baseDir, fullPath, entries);
|
|
72
|
-
} else if (stat.isFile()) {
|
|
73
|
-
if (stat.size > 50 * 1024 * 1024) continue;
|
|
74
|
-
const content = readFileSync(fullPath);
|
|
75
|
-
const sha256 = createHash("sha256").update(content).digest("hex");
|
|
76
|
-
entries.push({ path: relPath, sha256, size: stat.size });
|
|
77
|
-
}
|
|
78
|
-
}
|
|
79
|
-
}
|
|
80
|
-
function generateManifest() {
|
|
81
|
-
const paths = ldmPaths();
|
|
82
|
-
const entries = [];
|
|
83
|
-
scanDir(paths.root, paths.root, entries);
|
|
84
|
-
return {
|
|
85
|
-
version: 1,
|
|
86
|
-
generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
87
|
-
fileCount: entries.length,
|
|
88
|
-
entries
|
|
89
|
-
};
|
|
90
|
-
}
|
|
91
|
-
function compareManifest(coreManifest) {
|
|
92
|
-
const paths = ldmPaths();
|
|
93
|
-
const upsert = [];
|
|
94
|
-
const toDelete = [];
|
|
95
|
-
const localManifest = generateManifest();
|
|
96
|
-
const localMap = new Map(localManifest.entries.map((e) => [e.path, e]));
|
|
97
|
-
const coreMap = new Map(coreManifest.entries.map((e) => [e.path, e]));
|
|
98
|
-
for (const entry of coreManifest.entries) {
|
|
99
|
-
const local = localMap.get(entry.path);
|
|
100
|
-
if (!local || local.sha256 !== entry.sha256) {
|
|
101
|
-
upsert.push({ path: entry.path, sha256: entry.sha256 });
|
|
102
|
-
}
|
|
103
|
-
}
|
|
104
|
-
for (const entry of localManifest.entries) {
|
|
105
|
-
if (!coreMap.has(entry.path)) {
|
|
106
|
-
toDelete.push(entry.path);
|
|
107
|
-
}
|
|
108
|
-
}
|
|
109
|
-
return { upsert, delete: toDelete };
|
|
110
|
-
}
|
|
111
|
-
function loadFileSyncState() {
|
|
112
|
-
const statePath = resolveStatePath("file-sync-state.json");
|
|
113
|
-
try {
|
|
114
|
-
if (existsSync(statePath)) {
|
|
115
|
-
return JSON.parse(readFileSync(statePath, "utf-8"));
|
|
116
|
-
}
|
|
117
|
-
} catch {
|
|
118
|
-
}
|
|
119
|
-
return { lastSync: null, lastManifestHash: null, filesTransferred: 0, filesDeleted: 0 };
|
|
120
|
-
}
|
|
121
|
-
function saveFileSyncState(state) {
|
|
122
|
-
const writePath = stateWritePath("file-sync-state.json");
|
|
123
|
-
writeFileSync(writePath, JSON.stringify(state, null, 2));
|
|
124
|
-
}
|
|
125
|
-
async function pushFileSync() {
|
|
126
|
-
if (!RELAY_URL || !RELAY_TOKEN) {
|
|
127
|
-
throw new Error("CRYSTAL_RELAY_URL and CRYSTAL_RELAY_TOKEN must be set");
|
|
128
|
-
}
|
|
129
|
-
const relayKey = loadRelayKey();
|
|
130
|
-
const paths = ldmPaths();
|
|
131
|
-
const manifest = generateManifest();
|
|
132
|
-
const manifestJson = JSON.stringify(manifest.entries.map((e) => `${e.path}:${e.sha256}`));
|
|
133
|
-
const manifestHash = createHash("sha256").update(manifestJson).digest("hex");
|
|
134
|
-
const state = loadFileSyncState();
|
|
135
|
-
if (state.lastManifestHash === manifestHash) {
|
|
136
|
-
return { manifest: 0, files: 0 };
|
|
137
|
-
}
|
|
138
|
-
const encryptedManifest = encryptJSON(manifest, relayKey);
|
|
139
|
-
const manifestResp = await fetch(`${RELAY_URL}/drop/files`, {
|
|
140
|
-
method: "POST",
|
|
141
|
-
headers: {
|
|
142
|
-
"Authorization": `Bearer ${RELAY_TOKEN}`,
|
|
143
|
-
"Content-Type": "application/octet-stream",
|
|
144
|
-
"X-File-Type": "manifest"
|
|
145
|
-
},
|
|
146
|
-
body: JSON.stringify(encryptedManifest)
|
|
147
|
-
});
|
|
148
|
-
if (!manifestResp.ok) {
|
|
149
|
-
throw new Error(`Manifest push failed: ${manifestResp.status} ${await manifestResp.text()}`);
|
|
150
|
-
}
|
|
151
|
-
let filesPushed = 0;
|
|
152
|
-
for (const entry of manifest.entries) {
|
|
153
|
-
const fullPath = join(paths.root, entry.path);
|
|
154
|
-
if (!existsSync(fullPath)) continue;
|
|
155
|
-
const content = readFileSync(fullPath);
|
|
156
|
-
const encrypted = encrypt(content, relayKey);
|
|
157
|
-
const filePayload = JSON.stringify({
|
|
158
|
-
path: entry.path,
|
|
159
|
-
sha256: entry.sha256,
|
|
160
|
-
size: entry.size,
|
|
161
|
-
data: encrypted
|
|
162
|
-
});
|
|
163
|
-
const fileResp = await fetch(`${RELAY_URL}/drop/files`, {
|
|
164
|
-
method: "POST",
|
|
165
|
-
headers: {
|
|
166
|
-
"Authorization": `Bearer ${RELAY_TOKEN}`,
|
|
167
|
-
"Content-Type": "application/octet-stream",
|
|
168
|
-
"X-File-Type": "file"
|
|
169
|
-
},
|
|
170
|
-
body: filePayload
|
|
171
|
-
});
|
|
172
|
-
if (fileResp.ok) filesPushed++;
|
|
173
|
-
}
|
|
174
|
-
state.lastSync = (/* @__PURE__ */ new Date()).toISOString();
|
|
175
|
-
state.lastManifestHash = manifestHash;
|
|
176
|
-
state.filesTransferred += filesPushed;
|
|
177
|
-
saveFileSyncState(state);
|
|
178
|
-
return { manifest: manifest.fileCount, files: filesPushed };
|
|
179
|
-
}
|
|
180
|
-
async function pullFileSync() {
|
|
181
|
-
if (!RELAY_URL || !RELAY_TOKEN) {
|
|
182
|
-
throw new Error("CRYSTAL_RELAY_URL and CRYSTAL_RELAY_TOKEN must be set");
|
|
183
|
-
}
|
|
184
|
-
const relayKey = loadRelayKey();
|
|
185
|
-
const paths = ldmPaths();
|
|
186
|
-
const listResp = await fetch(`${RELAY_URL}/pickup/files`, {
|
|
187
|
-
headers: { "Authorization": `Bearer ${RELAY_TOKEN}` }
|
|
188
|
-
});
|
|
189
|
-
if (!listResp.ok) {
|
|
190
|
-
throw new Error(`File sync list failed: ${listResp.status} ${await listResp.text()}`);
|
|
191
|
-
}
|
|
192
|
-
const listData = await listResp.json();
|
|
193
|
-
if (listData.count === 0) {
|
|
194
|
-
return { imported: 0, deleted: 0 };
|
|
195
|
-
}
|
|
196
|
-
let coreManifest = null;
|
|
197
|
-
const fileBlobs = [];
|
|
198
|
-
for (const blob of listData.blobs) {
|
|
199
|
-
try {
|
|
200
|
-
const blobResp = await fetch(`${RELAY_URL}/pickup/files/${blob.id}`, {
|
|
201
|
-
headers: { "Authorization": `Bearer ${RELAY_TOKEN}` }
|
|
202
|
-
});
|
|
203
|
-
if (!blobResp.ok) continue;
|
|
204
|
-
const text = await blobResp.text();
|
|
205
|
-
const parsed = JSON.parse(text);
|
|
206
|
-
if (parsed.v !== void 0 && parsed.nonce !== void 0) {
|
|
207
|
-
try {
|
|
208
|
-
const manifest = decryptJSON(parsed, relayKey);
|
|
209
|
-
if (manifest.version && manifest.entries) {
|
|
210
|
-
coreManifest = manifest;
|
|
211
|
-
}
|
|
212
|
-
} catch {
|
|
213
|
-
}
|
|
214
|
-
} else if (parsed.path && parsed.data) {
|
|
215
|
-
fileBlobs.push({
|
|
216
|
-
id: blob.id,
|
|
217
|
-
path: parsed.path,
|
|
218
|
-
sha256: parsed.sha256,
|
|
219
|
-
data: parsed.data
|
|
220
|
-
});
|
|
221
|
-
}
|
|
222
|
-
await fetch(`${RELAY_URL}/confirm/files/${blob.id}`, {
|
|
223
|
-
method: "DELETE",
|
|
224
|
-
headers: { "Authorization": `Bearer ${RELAY_TOKEN}` }
|
|
225
|
-
});
|
|
226
|
-
} catch (err) {
|
|
227
|
-
process.stderr.write(`[file-sync] error processing blob ${blob.id}: ${err.message}
|
|
228
|
-
`);
|
|
229
|
-
}
|
|
230
|
-
}
|
|
231
|
-
if (!coreManifest) {
|
|
232
|
-
process.stderr.write("[file-sync] no manifest found in relay\n");
|
|
233
|
-
return { imported: 0, deleted: 0 };
|
|
234
|
-
}
|
|
235
|
-
const delta = compareManifest(coreManifest);
|
|
236
|
-
const fileBlobMap = new Map(fileBlobs.map((f) => [f.path, f]));
|
|
237
|
-
let imported = 0;
|
|
238
|
-
for (const entry of delta.upsert) {
|
|
239
|
-
const fileBlob = fileBlobMap.get(entry.path);
|
|
240
|
-
if (!fileBlob) {
|
|
241
|
-
continue;
|
|
242
|
-
}
|
|
243
|
-
try {
|
|
244
|
-
const content = decrypt(fileBlob.data, relayKey);
|
|
245
|
-
const actualHash = createHash("sha256").update(content).digest("hex");
|
|
246
|
-
if (actualHash !== entry.sha256) {
|
|
247
|
-
process.stderr.write(`[file-sync] hash mismatch for ${entry.path}, skipping
|
|
248
|
-
`);
|
|
249
|
-
continue;
|
|
250
|
-
}
|
|
251
|
-
const destPath = join(paths.root, entry.path);
|
|
252
|
-
mkdirSync(dirname(destPath), { recursive: true });
|
|
253
|
-
writeFileSync(destPath, content);
|
|
254
|
-
imported++;
|
|
255
|
-
} catch (err) {
|
|
256
|
-
process.stderr.write(`[file-sync] failed to write ${entry.path}: ${err.message}
|
|
257
|
-
`);
|
|
258
|
-
}
|
|
259
|
-
}
|
|
260
|
-
let deleted = 0;
|
|
261
|
-
for (const path of delta.delete) {
|
|
262
|
-
const destPath = join(paths.root, path);
|
|
263
|
-
if (existsSync(destPath)) {
|
|
264
|
-
try {
|
|
265
|
-
unlinkSync(destPath);
|
|
266
|
-
deleted++;
|
|
267
|
-
} catch (err) {
|
|
268
|
-
process.stderr.write(`[file-sync] failed to delete ${path}: ${err.message}
|
|
269
|
-
`);
|
|
270
|
-
}
|
|
271
|
-
}
|
|
272
|
-
}
|
|
273
|
-
const state = loadFileSyncState();
|
|
274
|
-
state.lastSync = (/* @__PURE__ */ new Date()).toISOString();
|
|
275
|
-
state.filesTransferred += imported;
|
|
276
|
-
state.filesDeleted += deleted;
|
|
277
|
-
saveFileSyncState(state);
|
|
278
|
-
return { imported, deleted };
|
|
279
|
-
}
|
|
280
|
-
|
|
281
|
-
export {
|
|
282
|
-
generateManifest,
|
|
283
|
-
compareManifest,
|
|
284
|
-
loadFileSyncState,
|
|
285
|
-
saveFileSyncState,
|
|
286
|
-
pushFileSync,
|
|
287
|
-
pullFileSync
|
|
288
|
-
};
|
package/dist/chunk-UNHVZB5G.js
DELETED
|
@@ -1,411 +0,0 @@
|
|
|
1
|
-
// src/core.ts
|
|
2
|
-
import * as lancedb from "@lancedb/lancedb";
|
|
3
|
-
import Database from "better-sqlite3";
|
|
4
|
-
import { readFileSync, existsSync, mkdirSync } from "fs";
|
|
5
|
-
import { execSync } from "child_process";
|
|
6
|
-
import { join } from "path";
|
|
7
|
-
import http from "http";
|
|
8
|
-
import https from "https";
|
|
9
|
-
async function embedOpenAI(texts, apiKey, model) {
|
|
10
|
-
return new Promise((resolve, reject) => {
|
|
11
|
-
const body = JSON.stringify({ input: texts, model });
|
|
12
|
-
const req = https.request({
|
|
13
|
-
hostname: "api.openai.com",
|
|
14
|
-
path: "/v1/embeddings",
|
|
15
|
-
method: "POST",
|
|
16
|
-
headers: {
|
|
17
|
-
"Content-Type": "application/json",
|
|
18
|
-
"Authorization": `Bearer ${apiKey}`,
|
|
19
|
-
"Content-Length": Buffer.byteLength(body)
|
|
20
|
-
},
|
|
21
|
-
timeout: 3e4
|
|
22
|
-
}, (res) => {
|
|
23
|
-
let data = "";
|
|
24
|
-
res.on("data", (chunk) => data += chunk);
|
|
25
|
-
res.on("end", () => {
|
|
26
|
-
if (res.statusCode !== 200) {
|
|
27
|
-
reject(new Error(`OpenAI API error ${res.statusCode}: ${data.slice(0, 200)}`));
|
|
28
|
-
return;
|
|
29
|
-
}
|
|
30
|
-
const parsed = JSON.parse(data);
|
|
31
|
-
resolve(parsed.data.map((d) => d.embedding));
|
|
32
|
-
});
|
|
33
|
-
});
|
|
34
|
-
req.on("error", reject);
|
|
35
|
-
req.on("timeout", () => {
|
|
36
|
-
req.destroy();
|
|
37
|
-
reject(new Error("OpenAI timeout"));
|
|
38
|
-
});
|
|
39
|
-
req.write(body);
|
|
40
|
-
req.end();
|
|
41
|
-
});
|
|
42
|
-
}
|
|
43
|
-
async function embedOllama(texts, host, model) {
|
|
44
|
-
const results = [];
|
|
45
|
-
for (const text of texts) {
|
|
46
|
-
const result = await new Promise((resolve, reject) => {
|
|
47
|
-
const url = new URL("/api/embeddings", host);
|
|
48
|
-
const body = JSON.stringify({ model, prompt: text });
|
|
49
|
-
const req = http.request({
|
|
50
|
-
hostname: url.hostname,
|
|
51
|
-
port: url.port,
|
|
52
|
-
path: url.pathname,
|
|
53
|
-
method: "POST",
|
|
54
|
-
headers: {
|
|
55
|
-
"Content-Type": "application/json",
|
|
56
|
-
"Content-Length": Buffer.byteLength(body)
|
|
57
|
-
},
|
|
58
|
-
timeout: 15e3
|
|
59
|
-
}, (res) => {
|
|
60
|
-
let data = "";
|
|
61
|
-
res.on("data", (chunk) => data += chunk);
|
|
62
|
-
res.on("end", () => {
|
|
63
|
-
if (res.statusCode !== 200) {
|
|
64
|
-
reject(new Error(`Ollama error ${res.statusCode}: ${data.slice(0, 200)}`));
|
|
65
|
-
return;
|
|
66
|
-
}
|
|
67
|
-
resolve(JSON.parse(data).embedding);
|
|
68
|
-
});
|
|
69
|
-
});
|
|
70
|
-
req.on("error", reject);
|
|
71
|
-
req.on("timeout", () => {
|
|
72
|
-
req.destroy();
|
|
73
|
-
reject(new Error("Ollama timeout"));
|
|
74
|
-
});
|
|
75
|
-
req.write(body);
|
|
76
|
-
req.end();
|
|
77
|
-
});
|
|
78
|
-
results.push(result);
|
|
79
|
-
}
|
|
80
|
-
return results;
|
|
81
|
-
}
|
|
82
|
-
async function embedGoogle(texts, apiKey, model) {
|
|
83
|
-
return new Promise((resolve, reject) => {
|
|
84
|
-
const body = JSON.stringify({
|
|
85
|
-
requests: texts.map((text) => ({ model: `models/${model}`, content: { parts: [{ text }] } }))
|
|
86
|
-
});
|
|
87
|
-
const req = https.request({
|
|
88
|
-
hostname: "generativelanguage.googleapis.com",
|
|
89
|
-
path: `/v1beta/models/${model}:batchEmbedContents?key=${apiKey}`,
|
|
90
|
-
method: "POST",
|
|
91
|
-
headers: {
|
|
92
|
-
"Content-Type": "application/json",
|
|
93
|
-
"Content-Length": Buffer.byteLength(body)
|
|
94
|
-
},
|
|
95
|
-
timeout: 3e4
|
|
96
|
-
}, (res) => {
|
|
97
|
-
let data = "";
|
|
98
|
-
res.on("data", (chunk) => data += chunk);
|
|
99
|
-
res.on("end", () => {
|
|
100
|
-
if (res.statusCode !== 200) {
|
|
101
|
-
reject(new Error(`Google API error ${res.statusCode}: ${data.slice(0, 200)}`));
|
|
102
|
-
return;
|
|
103
|
-
}
|
|
104
|
-
const parsed = JSON.parse(data);
|
|
105
|
-
resolve(parsed.embeddings.map((e) => e.values));
|
|
106
|
-
});
|
|
107
|
-
});
|
|
108
|
-
req.on("error", reject);
|
|
109
|
-
req.on("timeout", () => {
|
|
110
|
-
req.destroy();
|
|
111
|
-
reject(new Error("Google timeout"));
|
|
112
|
-
});
|
|
113
|
-
req.write(body);
|
|
114
|
-
req.end();
|
|
115
|
-
});
|
|
116
|
-
}
|
|
117
|
-
var Crystal = class {
|
|
118
|
-
config;
|
|
119
|
-
lanceDb = null;
|
|
120
|
-
sqliteDb = null;
|
|
121
|
-
chunksTable = null;
|
|
122
|
-
constructor(config) {
|
|
123
|
-
this.config = config;
|
|
124
|
-
if (!existsSync(config.dataDir)) {
|
|
125
|
-
mkdirSync(config.dataDir, { recursive: true });
|
|
126
|
-
}
|
|
127
|
-
}
|
|
128
|
-
// ── Initialization ──
|
|
129
|
-
async init() {
|
|
130
|
-
const lanceDir = join(this.config.dataDir, "lance");
|
|
131
|
-
const sqlitePath = join(this.config.dataDir, "crystal.db");
|
|
132
|
-
if (!existsSync(lanceDir)) mkdirSync(lanceDir, { recursive: true });
|
|
133
|
-
this.lanceDb = await lancedb.connect(lanceDir);
|
|
134
|
-
this.sqliteDb = new Database(sqlitePath);
|
|
135
|
-
this.sqliteDb.pragma("journal_mode = WAL");
|
|
136
|
-
this.initSqliteTables();
|
|
137
|
-
await this.initLanceTables();
|
|
138
|
-
}
|
|
139
|
-
initSqliteTables() {
|
|
140
|
-
const db = this.sqliteDb;
|
|
141
|
-
db.exec(`
|
|
142
|
-
CREATE TABLE IF NOT EXISTS sources (
|
|
143
|
-
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
144
|
-
type TEXT NOT NULL,
|
|
145
|
-
uri TEXT NOT NULL,
|
|
146
|
-
title TEXT,
|
|
147
|
-
agent_id TEXT NOT NULL,
|
|
148
|
-
metadata TEXT DEFAULT '{}',
|
|
149
|
-
ingested_at TEXT NOT NULL,
|
|
150
|
-
chunk_count INTEGER DEFAULT 0
|
|
151
|
-
);
|
|
152
|
-
|
|
153
|
-
CREATE TABLE IF NOT EXISTS capture_state (
|
|
154
|
-
agent_id TEXT NOT NULL,
|
|
155
|
-
source_id TEXT NOT NULL,
|
|
156
|
-
last_message_count INTEGER DEFAULT 0,
|
|
157
|
-
capture_count INTEGER DEFAULT 0,
|
|
158
|
-
last_capture_at TEXT,
|
|
159
|
-
PRIMARY KEY (agent_id, source_id)
|
|
160
|
-
);
|
|
161
|
-
|
|
162
|
-
CREATE TABLE IF NOT EXISTS memories (
|
|
163
|
-
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
164
|
-
text TEXT NOT NULL,
|
|
165
|
-
category TEXT NOT NULL DEFAULT 'fact',
|
|
166
|
-
confidence REAL NOT NULL DEFAULT 1.0,
|
|
167
|
-
source_ids TEXT DEFAULT '[]',
|
|
168
|
-
status TEXT NOT NULL DEFAULT 'active',
|
|
169
|
-
created_at TEXT NOT NULL,
|
|
170
|
-
updated_at TEXT NOT NULL
|
|
171
|
-
);
|
|
172
|
-
|
|
173
|
-
CREATE TABLE IF NOT EXISTS entities (
|
|
174
|
-
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
175
|
-
name TEXT NOT NULL UNIQUE,
|
|
176
|
-
type TEXT NOT NULL DEFAULT 'concept',
|
|
177
|
-
description TEXT,
|
|
178
|
-
properties TEXT DEFAULT '{}',
|
|
179
|
-
created_at TEXT NOT NULL,
|
|
180
|
-
updated_at TEXT NOT NULL
|
|
181
|
-
);
|
|
182
|
-
|
|
183
|
-
CREATE TABLE IF NOT EXISTS relationships (
|
|
184
|
-
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
185
|
-
source_id INTEGER NOT NULL REFERENCES entities(id),
|
|
186
|
-
target_id INTEGER NOT NULL REFERENCES entities(id),
|
|
187
|
-
type TEXT NOT NULL,
|
|
188
|
-
description TEXT,
|
|
189
|
-
weight REAL DEFAULT 1.0,
|
|
190
|
-
valid_from TEXT NOT NULL,
|
|
191
|
-
valid_until TEXT,
|
|
192
|
-
created_at TEXT NOT NULL
|
|
193
|
-
);
|
|
194
|
-
|
|
195
|
-
CREATE INDEX IF NOT EXISTS idx_sources_agent ON sources(agent_id);
|
|
196
|
-
CREATE INDEX IF NOT EXISTS idx_memories_status ON memories(status);
|
|
197
|
-
CREATE INDEX IF NOT EXISTS idx_entities_name ON entities(name);
|
|
198
|
-
CREATE INDEX IF NOT EXISTS idx_relationships_source ON relationships(source_id);
|
|
199
|
-
CREATE INDEX IF NOT EXISTS idx_relationships_target ON relationships(target_id);
|
|
200
|
-
`);
|
|
201
|
-
}
|
|
202
|
-
async initLanceTables() {
|
|
203
|
-
const db = this.lanceDb;
|
|
204
|
-
const tableNames = await db.tableNames();
|
|
205
|
-
if (tableNames.includes("chunks")) {
|
|
206
|
-
this.chunksTable = await db.openTable("chunks");
|
|
207
|
-
}
|
|
208
|
-
}
|
|
209
|
-
// ── Embedding ──
|
|
210
|
-
async embed(texts) {
|
|
211
|
-
if (texts.length === 0) return [];
|
|
212
|
-
const cfg = this.config;
|
|
213
|
-
switch (cfg.embeddingProvider) {
|
|
214
|
-
case "openai":
|
|
215
|
-
if (!cfg.openaiApiKey) throw new Error("OpenAI API key required");
|
|
216
|
-
return embedOpenAI(texts, cfg.openaiApiKey, cfg.openaiModel || "text-embedding-3-small");
|
|
217
|
-
case "ollama":
|
|
218
|
-
return embedOllama(texts, cfg.ollamaHost || "http://localhost:11434", cfg.ollamaModel || "nomic-embed-text");
|
|
219
|
-
case "google":
|
|
220
|
-
if (!cfg.googleApiKey) throw new Error("Google API key required");
|
|
221
|
-
return embedGoogle(texts, cfg.googleApiKey, cfg.googleModel || "text-embedding-004");
|
|
222
|
-
default:
|
|
223
|
-
throw new Error(`Unknown embedding provider: ${cfg.embeddingProvider}`);
|
|
224
|
-
}
|
|
225
|
-
}
|
|
226
|
-
// ── Chunking ──
|
|
227
|
-
chunkText(text, targetTokens = 400, overlapTokens = 80) {
|
|
228
|
-
const targetChars = targetTokens * 4;
|
|
229
|
-
const overlapChars = overlapTokens * 4;
|
|
230
|
-
const chunks = [];
|
|
231
|
-
let start = 0;
|
|
232
|
-
while (start < text.length) {
|
|
233
|
-
let end = Math.min(start + targetChars, text.length);
|
|
234
|
-
if (end < text.length) {
|
|
235
|
-
const minBreak = start + Math.floor(targetChars * 0.5);
|
|
236
|
-
const paraBreak = text.lastIndexOf("\n\n", end);
|
|
237
|
-
if (paraBreak > minBreak) {
|
|
238
|
-
end = paraBreak;
|
|
239
|
-
} else {
|
|
240
|
-
const sentBreak = text.lastIndexOf(". ", end);
|
|
241
|
-
if (sentBreak > minBreak) {
|
|
242
|
-
end = sentBreak + 1;
|
|
243
|
-
}
|
|
244
|
-
}
|
|
245
|
-
}
|
|
246
|
-
const chunk = text.slice(start, end).trim();
|
|
247
|
-
if (chunk.length > 0) chunks.push(chunk);
|
|
248
|
-
if (end >= text.length) break;
|
|
249
|
-
start = end - overlapChars;
|
|
250
|
-
if (start <= (chunks.length > 0 ? end - targetChars : 0)) {
|
|
251
|
-
start = end;
|
|
252
|
-
}
|
|
253
|
-
}
|
|
254
|
-
return chunks;
|
|
255
|
-
}
|
|
256
|
-
// ── Ingest ──
|
|
257
|
-
async ingest(chunks) {
|
|
258
|
-
if (chunks.length === 0) return 0;
|
|
259
|
-
const texts = chunks.map((c) => c.text);
|
|
260
|
-
const embeddings = await this.embed(texts);
|
|
261
|
-
const records = chunks.map((chunk, i) => ({
|
|
262
|
-
text: chunk.text,
|
|
263
|
-
vector: embeddings[i],
|
|
264
|
-
role: chunk.role,
|
|
265
|
-
source_type: chunk.source_type,
|
|
266
|
-
source_id: chunk.source_id,
|
|
267
|
-
agent_id: chunk.agent_id,
|
|
268
|
-
token_count: chunk.token_count,
|
|
269
|
-
created_at: chunk.created_at || (/* @__PURE__ */ new Date()).toISOString()
|
|
270
|
-
}));
|
|
271
|
-
if (!this.chunksTable) {
|
|
272
|
-
this.chunksTable = await this.lanceDb.createTable("chunks", records);
|
|
273
|
-
} else {
|
|
274
|
-
await this.chunksTable.add(records);
|
|
275
|
-
}
|
|
276
|
-
return records.length;
|
|
277
|
-
}
|
|
278
|
-
// ── Search ──
|
|
279
|
-
async search(query, limit = 5, filter) {
|
|
280
|
-
if (!this.chunksTable) return [];
|
|
281
|
-
const [embedding] = await this.embed([query]);
|
|
282
|
-
let queryBuilder = this.chunksTable.vectorSearch(embedding).distanceType("cosine").limit(limit);
|
|
283
|
-
if (filter?.agent_id) {
|
|
284
|
-
queryBuilder = queryBuilder.where(`agent_id = '${filter.agent_id}'`);
|
|
285
|
-
}
|
|
286
|
-
if (filter?.source_type) {
|
|
287
|
-
queryBuilder = queryBuilder.where(`source_type = '${filter.source_type}'`);
|
|
288
|
-
}
|
|
289
|
-
const results = await queryBuilder.toArray();
|
|
290
|
-
return results.map((row) => ({
|
|
291
|
-
text: row.text,
|
|
292
|
-
role: row.role,
|
|
293
|
-
score: row._distance != null ? 1 - row._distance : 0,
|
|
294
|
-
source_type: row.source_type,
|
|
295
|
-
source_id: row.source_id,
|
|
296
|
-
agent_id: row.agent_id,
|
|
297
|
-
created_at: row.created_at
|
|
298
|
-
}));
|
|
299
|
-
}
|
|
300
|
-
// ── Remember (explicit fact storage) ──
|
|
301
|
-
async remember(text, category = "fact") {
|
|
302
|
-
const db = this.sqliteDb;
|
|
303
|
-
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
304
|
-
const stmt = db.prepare(`
|
|
305
|
-
INSERT INTO memories (text, category, confidence, source_ids, status, created_at, updated_at)
|
|
306
|
-
VALUES (?, ?, 1.0, '[]', 'active', ?, ?)
|
|
307
|
-
`);
|
|
308
|
-
const result = stmt.run(text, category, now, now);
|
|
309
|
-
await this.ingest([{
|
|
310
|
-
text,
|
|
311
|
-
role: "system",
|
|
312
|
-
source_type: "manual",
|
|
313
|
-
source_id: `memory:${result.lastInsertRowid}`,
|
|
314
|
-
agent_id: "system",
|
|
315
|
-
token_count: Math.ceil(text.length / 4),
|
|
316
|
-
created_at: now
|
|
317
|
-
}]);
|
|
318
|
-
return result.lastInsertRowid;
|
|
319
|
-
}
|
|
320
|
-
// ── Forget (deprecate a memory) ──
|
|
321
|
-
forget(memoryId) {
|
|
322
|
-
const db = this.sqliteDb;
|
|
323
|
-
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
324
|
-
const result = db.prepare(`
|
|
325
|
-
UPDATE memories SET status = 'deprecated', updated_at = ? WHERE id = ? AND status = 'active'
|
|
326
|
-
`).run(now, memoryId);
|
|
327
|
-
return result.changes > 0;
|
|
328
|
-
}
|
|
329
|
-
// ── Status ──
|
|
330
|
-
async status() {
|
|
331
|
-
const db = this.sqliteDb;
|
|
332
|
-
let chunks = 0;
|
|
333
|
-
let oldestChunk = null;
|
|
334
|
-
let newestChunk = null;
|
|
335
|
-
const agents = [];
|
|
336
|
-
if (this.chunksTable) {
|
|
337
|
-
chunks = await this.chunksTable.countRows();
|
|
338
|
-
try {
|
|
339
|
-
const sample = await this.chunksTable.search([]).limit(1).toArray();
|
|
340
|
-
} catch {
|
|
341
|
-
}
|
|
342
|
-
}
|
|
343
|
-
const memories = db.prepare("SELECT COUNT(*) as count FROM memories WHERE status = ?").get("active")?.count || 0;
|
|
344
|
-
const sources = db.prepare("SELECT COUNT(*) as count FROM sources").get()?.count || 0;
|
|
345
|
-
const agentRows = db.prepare("SELECT DISTINCT agent_id FROM sources").all();
|
|
346
|
-
agents.push(...agentRows.map((r) => r.agent_id));
|
|
347
|
-
return {
|
|
348
|
-
chunks,
|
|
349
|
-
memories,
|
|
350
|
-
sources,
|
|
351
|
-
agents,
|
|
352
|
-
oldestChunk,
|
|
353
|
-
newestChunk,
|
|
354
|
-
embeddingProvider: this.config.embeddingProvider,
|
|
355
|
-
dataDir: this.config.dataDir
|
|
356
|
-
};
|
|
357
|
-
}
|
|
358
|
-
// ── Capture State (for incremental ingestion) ──
|
|
359
|
-
getCaptureState(agentId, sourceId) {
|
|
360
|
-
const db = this.sqliteDb;
|
|
361
|
-
const row = db.prepare("SELECT last_message_count, capture_count FROM capture_state WHERE agent_id = ? AND source_id = ?").get(agentId, sourceId);
|
|
362
|
-
return row || { lastMessageCount: 0, captureCount: 0 };
|
|
363
|
-
}
|
|
364
|
-
setCaptureState(agentId, sourceId, messageCount, captureCount) {
|
|
365
|
-
const db = this.sqliteDb;
|
|
366
|
-
db.prepare(`
|
|
367
|
-
INSERT OR REPLACE INTO capture_state (agent_id, source_id, last_message_count, capture_count, last_capture_at)
|
|
368
|
-
VALUES (?, ?, ?, ?, ?)
|
|
369
|
-
`).run(agentId, sourceId, messageCount, captureCount, (/* @__PURE__ */ new Date()).toISOString());
|
|
370
|
-
}
|
|
371
|
-
// ── Cleanup ──
|
|
372
|
-
close() {
|
|
373
|
-
this.sqliteDb?.close();
|
|
374
|
-
}
|
|
375
|
-
};
|
|
376
|
-
function resolveConfig(overrides) {
|
|
377
|
-
const openclawHome = process.env.OPENCLAW_HOME || join(process.env.HOME || "/Users/lesa", ".openclaw");
|
|
378
|
-
const dataDir = overrides?.dataDir || join(openclawHome, "memory-crystal");
|
|
379
|
-
let openaiApiKey = overrides?.openaiApiKey || process.env.OPENAI_API_KEY;
|
|
380
|
-
if (!openaiApiKey) {
|
|
381
|
-
try {
|
|
382
|
-
const saTokenPath = join(openclawHome, "secrets", "op-sa-token");
|
|
383
|
-
if (existsSync(saTokenPath)) {
|
|
384
|
-
const saToken = readFileSync(saTokenPath, "utf8").trim();
|
|
385
|
-
openaiApiKey = execSync('op read "op://Agent Secrets/OpenAI API/api key"', {
|
|
386
|
-
encoding: "utf8",
|
|
387
|
-
env: { ...process.env, OP_SERVICE_ACCOUNT_TOKEN: saToken },
|
|
388
|
-
timeout: 1e4
|
|
389
|
-
}).trim();
|
|
390
|
-
}
|
|
391
|
-
} catch {
|
|
392
|
-
}
|
|
393
|
-
}
|
|
394
|
-
return {
|
|
395
|
-
dataDir,
|
|
396
|
-
embeddingProvider: overrides?.embeddingProvider || process.env.CRYSTAL_EMBEDDING_PROVIDER || "openai",
|
|
397
|
-
openaiApiKey,
|
|
398
|
-
openaiModel: overrides?.openaiModel || process.env.CRYSTAL_OPENAI_MODEL || "text-embedding-3-small",
|
|
399
|
-
ollamaHost: overrides?.ollamaHost || process.env.CRYSTAL_OLLAMA_HOST || "http://localhost:11434",
|
|
400
|
-
ollamaModel: overrides?.ollamaModel || process.env.CRYSTAL_OLLAMA_MODEL || "nomic-embed-text",
|
|
401
|
-
googleApiKey: overrides?.googleApiKey || process.env.GOOGLE_API_KEY,
|
|
402
|
-
googleModel: overrides?.googleModel || process.env.CRYSTAL_GOOGLE_MODEL || "text-embedding-004",
|
|
403
|
-
remoteUrl: overrides?.remoteUrl || process.env.CRYSTAL_REMOTE_URL,
|
|
404
|
-
remoteToken: overrides?.remoteToken || process.env.CRYSTAL_REMOTE_TOKEN
|
|
405
|
-
};
|
|
406
|
-
}
|
|
407
|
-
|
|
408
|
-
export {
|
|
409
|
-
Crystal,
|
|
410
|
-
resolveConfig
|
|
411
|
-
};
|