@plur-ai/core 0.9.3 → 0.9.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/{chunk-3ZPTRZE3.js → chunk-EQPTF4JZ.js} +11 -1
- package/dist/{chunk-IGGMRXAB.js → chunk-ZY4R3VEG.js} +52 -12
- package/dist/embeddings-3EXLC3EH.js +21 -0
- package/dist/index.d.ts +306 -1
- package/dist/index.js +345 -42
- package/dist/{learn-async-R5KZ5EWF.js → learn-async-IISWD7HC.js} +1 -2
- package/package.json +1 -1
- package/dist/chunk-2ZDO52B4.js +0 -52
- package/dist/embeddings-ZRT6IRPA.js +0 -14
- package/dist/onnxruntime_binding-5QEF3SUC.node +0 -0
- package/dist/onnxruntime_binding-BKPKNEGC.node +0 -0
- package/dist/onnxruntime_binding-FMOXGIUT.node +0 -0
- package/dist/onnxruntime_binding-OI2KMXC5.node +0 -0
- package/dist/onnxruntime_binding-UX44MLAZ.node +0 -0
- package/dist/onnxruntime_binding-Y2W7N7WY.node +0 -0
- package/dist/transformers.node-PH5YK5EA.js +0 -46777
package/dist/index.js
CHANGED
|
@@ -1,12 +1,15 @@
|
|
|
1
1
|
import {
|
|
2
2
|
computeIdf,
|
|
3
|
+
embedderStatus,
|
|
3
4
|
embeddingSearch,
|
|
4
5
|
embeddingSearchWithScores,
|
|
5
6
|
engramSearchText,
|
|
6
7
|
ftsScore,
|
|
7
8
|
ftsTokenize,
|
|
8
|
-
|
|
9
|
-
|
|
9
|
+
resetEmbedder,
|
|
10
|
+
searchEngrams,
|
|
11
|
+
setEmbeddingsEnabled
|
|
12
|
+
} from "./chunk-ZY4R3VEG.js";
|
|
10
13
|
import {
|
|
11
14
|
EngramSchemaPassthrough,
|
|
12
15
|
appendHistory,
|
|
@@ -26,19 +29,18 @@ import {
|
|
|
26
29
|
readHistoryForEngram,
|
|
27
30
|
saveEngrams,
|
|
28
31
|
storePrefix
|
|
29
|
-
} from "./chunk-
|
|
32
|
+
} from "./chunk-EQPTF4JZ.js";
|
|
30
33
|
import {
|
|
31
34
|
atomicWrite,
|
|
32
35
|
getSyncStatus,
|
|
33
36
|
sync,
|
|
34
37
|
withLock
|
|
35
38
|
} from "./chunk-PRK3B7WR.js";
|
|
36
|
-
import "./chunk-2ZDO52B4.js";
|
|
37
39
|
|
|
38
40
|
// src/index.ts
|
|
39
41
|
import * as fs4 from "fs";
|
|
40
|
-
import
|
|
41
|
-
import
|
|
42
|
+
import { tmpdir } from "os";
|
|
43
|
+
import { join as join5, dirname as dirname2, basename as basename2 } from "path";
|
|
42
44
|
import yaml6 from "js-yaml";
|
|
43
45
|
|
|
44
46
|
// src/storage.ts
|
|
@@ -244,6 +246,9 @@ var StorageConfigSchema = z.object({
|
|
|
244
246
|
backend: z.enum(["yaml", "sqlite"]).default("yaml"),
|
|
245
247
|
path: z.string().optional()
|
|
246
248
|
}).partial();
|
|
249
|
+
var EmbeddingsConfigSchema = z.object({
|
|
250
|
+
enabled: z.boolean().default(true)
|
|
251
|
+
}).partial();
|
|
247
252
|
var PlurConfigSchema = z.object({
|
|
248
253
|
auto_learn: z.boolean().default(true),
|
|
249
254
|
auto_capture: z.boolean().default(true),
|
|
@@ -261,6 +266,7 @@ var PlurConfigSchema = z.object({
|
|
|
261
266
|
allow_secrets: z.boolean().default(false),
|
|
262
267
|
index: z.boolean().default(true),
|
|
263
268
|
storage: StorageConfigSchema.default({}),
|
|
269
|
+
embeddings: EmbeddingsConfigSchema.default({}),
|
|
264
270
|
stores: z.array(StoreEntrySchema).default([]),
|
|
265
271
|
llm: LlmTierConfigSchema.default({}),
|
|
266
272
|
profile: ProfileConfigSchema.default({}),
|
|
@@ -450,6 +456,7 @@ var DEFAULT_MAX_TOKENS = 8e3;
|
|
|
450
456
|
var DEFAULT_MIN_RELEVANCE = 0.3;
|
|
451
457
|
var MAX_PER_PACK = 5;
|
|
452
458
|
var MAX_PER_DOMAIN = 10;
|
|
459
|
+
var PINNED_TOKEN_BUDGET_RATIO = 0.5;
|
|
453
460
|
var DIP19_CONSIDER_MAX = 5;
|
|
454
461
|
var DIP19_CONSIDER_BUDGET = 200;
|
|
455
462
|
function getPackMetadata(manifest) {
|
|
@@ -528,7 +535,11 @@ function scoreEngram(engram, promptLower, promptWords, packMatchTerms, scopeFilt
|
|
|
528
535
|
for (const word of promptWords) {
|
|
529
536
|
if (statementWords.has(word)) termHits += 0.5;
|
|
530
537
|
}
|
|
531
|
-
|
|
538
|
+
const isPinned = engram.pinned === true;
|
|
539
|
+
if (termHits === 0 && !isPinned) return 0;
|
|
540
|
+
if (termHits === 0 && isPinned) {
|
|
541
|
+
termHits = 0.5;
|
|
542
|
+
}
|
|
532
543
|
let rs = isPack ? engram.activation.retrieval_strength : decayedStrength(engram.activation.retrieval_strength, daysSince(engram.activation.last_accessed));
|
|
533
544
|
if (!isPack) {
|
|
534
545
|
const fb = engram.feedback_signals;
|
|
@@ -543,6 +554,7 @@ function scoreEngram(engram, promptLower, promptWords, packMatchTerms, scopeFilt
|
|
|
543
554
|
else if (netFeedback < 0) score *= Math.max(1 + netFeedback * 0.1, 0.5);
|
|
544
555
|
}
|
|
545
556
|
if (engram.consolidated) score *= 1.1;
|
|
557
|
+
if (isPinned) score *= 2;
|
|
546
558
|
const emotionalWeight = engram.episodic?.emotional_weight ?? 5;
|
|
547
559
|
score *= 1 + (emotionalWeight - 5) * 0.04;
|
|
548
560
|
return score;
|
|
@@ -552,7 +564,21 @@ function fillTokenBudget(scored, maxTokens) {
|
|
|
552
564
|
const packCounts = /* @__PURE__ */ new Map();
|
|
553
565
|
const domainCounts = /* @__PURE__ */ new Map();
|
|
554
566
|
let tokensUsed = 0;
|
|
555
|
-
|
|
567
|
+
const pinned = scored.filter((e) => e.pinned === true);
|
|
568
|
+
const unpinned = scored.filter((e) => e.pinned !== true);
|
|
569
|
+
const pinnedBudget = Math.floor(maxTokens * PINNED_TOKEN_BUDGET_RATIO);
|
|
570
|
+
for (const engram of pinned) {
|
|
571
|
+
const cost = estimateTokens(engram);
|
|
572
|
+
if (tokensUsed + cost > maxTokens) continue;
|
|
573
|
+
if (tokensUsed + cost > pinnedBudget) continue;
|
|
574
|
+
result.push(engram);
|
|
575
|
+
tokensUsed += cost;
|
|
576
|
+
const pack = engram.pack ?? "__personal__";
|
|
577
|
+
packCounts.set(pack, (packCounts.get(pack) ?? 0) + 1);
|
|
578
|
+
const topDomain = (engram.domain ?? "__none__").split(".")[0];
|
|
579
|
+
domainCounts.set(topDomain, (domainCounts.get(topDomain) ?? 0) + 1);
|
|
580
|
+
}
|
|
581
|
+
for (const engram of unpinned) {
|
|
556
582
|
const cost = estimateTokens(engram);
|
|
557
583
|
if (tokensUsed + cost > maxTokens) continue;
|
|
558
584
|
const pack = engram.pack ?? "__personal__";
|
|
@@ -586,7 +612,7 @@ function selectAndSpread(ctx, personalEngrams, packs, config, embeddingBoosts) {
|
|
|
586
612
|
engramMap.set(engram.id, engram);
|
|
587
613
|
let raw = scoreEngram(engram, promptLower, promptWords, [], ctx.scope, false);
|
|
588
614
|
const embBoost = embeddingBoosts?.get(engram.id) ?? 0;
|
|
589
|
-
if (raw === 0 && embBoost > 0.
|
|
615
|
+
if (raw === 0 && embBoost > 0.5) {
|
|
590
616
|
raw = embBoost * 2;
|
|
591
617
|
} else if (raw > 0 && embBoost > 0) {
|
|
592
618
|
raw += embBoost;
|
|
@@ -611,7 +637,7 @@ function selectAndSpread(ctx, personalEngrams, packs, config, embeddingBoosts) {
|
|
|
611
637
|
engramMap.set(engram.id, engram);
|
|
612
638
|
let raw = scoreEngram(engram, promptLower, promptWords, matchTerms, ctx.scope, true);
|
|
613
639
|
const embBoost = embeddingBoosts?.get(engram.id) ?? 0;
|
|
614
|
-
if (raw === 0 && embBoost > 0.
|
|
640
|
+
if (raw === 0 && embBoost > 0.5) {
|
|
615
641
|
raw = embBoost * 2;
|
|
616
642
|
} else if (raw > 0 && embBoost > 0) {
|
|
617
643
|
raw += embBoost;
|
|
@@ -631,7 +657,7 @@ function selectAndSpread(ctx, personalEngrams, packs, config, embeddingBoosts) {
|
|
|
631
657
|
aBoosts.set(e.id, aBoost);
|
|
632
658
|
e.score = e.keyword_match + aBoost;
|
|
633
659
|
}
|
|
634
|
-
const filtered = scored.filter((s) => s.score >= minRelevance);
|
|
660
|
+
const filtered = scored.filter((s) => s.pinned === true || s.score >= minRelevance);
|
|
635
661
|
filtered.sort((a, b) => b.score - a.score);
|
|
636
662
|
const { selected: directives, tokens_used: directiveTokens } = fillTokenBudget(filtered, maxTokens);
|
|
637
663
|
const directiveIds = new Set(directives.map((e) => e.id));
|
|
@@ -772,8 +798,8 @@ function generateEpisodeId() {
|
|
|
772
798
|
const rand = Math.random().toString(36).slice(2, 6);
|
|
773
799
|
return `EP-${ts}-${rand}`;
|
|
774
800
|
}
|
|
775
|
-
function captureEpisode(
|
|
776
|
-
const episodes = loadEpisodes(
|
|
801
|
+
function captureEpisode(path3, summary, context) {
|
|
802
|
+
const episodes = loadEpisodes(path3);
|
|
777
803
|
const episode = {
|
|
778
804
|
id: generateEpisodeId(),
|
|
779
805
|
summary,
|
|
@@ -784,11 +810,11 @@ function captureEpisode(path4, summary, context) {
|
|
|
784
810
|
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
785
811
|
};
|
|
786
812
|
episodes.push(episode);
|
|
787
|
-
atomicWrite(
|
|
813
|
+
atomicWrite(path3, yaml2.dump(episodes, { lineWidth: 120, noRefs: true }));
|
|
788
814
|
return episode;
|
|
789
815
|
}
|
|
790
|
-
function queryTimeline(
|
|
791
|
-
let episodes = loadEpisodes(
|
|
816
|
+
function queryTimeline(path3, query) {
|
|
817
|
+
let episodes = loadEpisodes(path3);
|
|
792
818
|
if (query?.since) episodes = episodes.filter((e) => new Date(e.timestamp) >= query.since);
|
|
793
819
|
if (query?.until) episodes = episodes.filter((e) => new Date(e.timestamp) <= query.until);
|
|
794
820
|
if (query?.agent) episodes = episodes.filter((e) => e.agent === query.agent);
|
|
@@ -799,10 +825,10 @@ function queryTimeline(path4, query) {
|
|
|
799
825
|
}
|
|
800
826
|
return episodes;
|
|
801
827
|
}
|
|
802
|
-
function loadEpisodes(
|
|
803
|
-
if (!existsSync4(
|
|
828
|
+
function loadEpisodes(path3) {
|
|
829
|
+
if (!existsSync4(path3)) return [];
|
|
804
830
|
try {
|
|
805
|
-
const raw = yaml2.load(readFileSync2(
|
|
831
|
+
const raw = yaml2.load(readFileSync2(path3, "utf8"));
|
|
806
832
|
return Array.isArray(raw) ? raw : [];
|
|
807
833
|
} catch {
|
|
808
834
|
return [];
|
|
@@ -903,7 +929,13 @@ function isAggregationQuery(query) {
|
|
|
903
929
|
return AGGREGATION_PATTERNS.some((p) => p.test(query));
|
|
904
930
|
}
|
|
905
931
|
async function hybridSearch(engrams, query, limit, storagePath) {
|
|
906
|
-
|
|
932
|
+
const result = await hybridSearchWithMeta(engrams, query, limit, storagePath);
|
|
933
|
+
return result.engrams;
|
|
934
|
+
}
|
|
935
|
+
async function hybridSearchWithMeta(engrams, query, limit, storagePath) {
|
|
936
|
+
if (engrams.length === 0) {
|
|
937
|
+
return { engrams: [], mode: "hybrid", embedderError: null };
|
|
938
|
+
}
|
|
907
939
|
const exhaustive = isAggregationQuery(query);
|
|
908
940
|
const effectiveLimit = exhaustive ? Math.max(limit, 50) : limit;
|
|
909
941
|
const bm25Limit = Math.min(engrams.length, exhaustive ? effectiveLimit * 5 : effectiveLimit * 3);
|
|
@@ -912,8 +944,23 @@ async function hybridSearch(engrams, query, limit, storagePath) {
|
|
|
912
944
|
Promise.resolve(searchEngrams(engrams, query, bm25Limit)),
|
|
913
945
|
embeddingSearch(engrams, query, embLimit, storagePath)
|
|
914
946
|
]);
|
|
947
|
+
const status = embedderStatus();
|
|
948
|
+
let mode;
|
|
949
|
+
let embedderError = null;
|
|
950
|
+
if (status.disabled) {
|
|
951
|
+
mode = "bm25-only";
|
|
952
|
+
} else if (!status.available || embResults.length === 0 && !!status.lastError) {
|
|
953
|
+
mode = "hybrid-degraded";
|
|
954
|
+
embedderError = status.lastError;
|
|
955
|
+
} else {
|
|
956
|
+
mode = "hybrid";
|
|
957
|
+
}
|
|
915
958
|
const merged = rrfMerge([bm25Results, embResults]);
|
|
916
|
-
return
|
|
959
|
+
return {
|
|
960
|
+
engrams: merged.slice(0, effectiveLimit),
|
|
961
|
+
mode,
|
|
962
|
+
embedderError
|
|
963
|
+
};
|
|
917
964
|
}
|
|
918
965
|
|
|
919
966
|
// src/query-expansion.ts
|
|
@@ -1585,7 +1632,7 @@ async function computeSimilarityMatrix(templates) {
|
|
|
1585
1632
|
const n = templates.length;
|
|
1586
1633
|
const matrix = Array.from({ length: n }, () => Array(n).fill(0));
|
|
1587
1634
|
try {
|
|
1588
|
-
const { embed, cosineSimilarity } = await import("./embeddings-
|
|
1635
|
+
const { embed, cosineSimilarity } = await import("./embeddings-3EXLC3EH.js");
|
|
1589
1636
|
const embeddings = [];
|
|
1590
1637
|
for (const t of templates) {
|
|
1591
1638
|
embeddings.push(await embed(t));
|
|
@@ -2936,6 +2983,192 @@ function isNewer(a, b) {
|
|
|
2936
2983
|
return false;
|
|
2937
2984
|
}
|
|
2938
2985
|
|
|
2986
|
+
// src/schemas/capsule.ts
|
|
2987
|
+
import { z as z3 } from "zod";
|
|
2988
|
+
var CAPSULE_MAGIC = Buffer.from([80, 76, 85, 82]);
|
|
2989
|
+
var CAPSULE_MAGIC_HEX = "50 4c 55 52";
|
|
2990
|
+
var FORMAT_VERSION_V1 = 1;
|
|
2991
|
+
var SUPPORTED_FORMAT_VERSIONS = [FORMAT_VERSION_V1];
|
|
2992
|
+
var CAPSULE_FLAGS = {
|
|
2993
|
+
SIGNED: 1 << 0,
|
|
2994
|
+
COMPRESSED: 1 << 1
|
|
2995
|
+
};
|
|
2996
|
+
var CAPSULE_FLAG_RESERVED_MASK = 65532;
|
|
2997
|
+
var PREAMBLE_LEN = 12;
|
|
2998
|
+
var CAPSULE_SIZE_LIMITS = {
|
|
2999
|
+
SOFT_BYTES: 100 * 1024 * 1024,
|
|
3000
|
+
HARD_BYTES: 1024 * 1024 * 1024
|
|
3001
|
+
};
|
|
3002
|
+
var ED25519_SIG_LEN = 64;
|
|
3003
|
+
var ManifestSummarySchema = z3.object({
|
|
3004
|
+
name: z3.string().min(1),
|
|
3005
|
+
version: z3.string().min(1),
|
|
3006
|
+
creator: z3.string().optional(),
|
|
3007
|
+
engram_count: z3.number().int().min(0),
|
|
3008
|
+
domain: z3.string().optional(),
|
|
3009
|
+
license: z3.string().default("cc-by-sa-4.0")
|
|
3010
|
+
});
|
|
3011
|
+
var PayloadDescriptorSchema = z3.object({
|
|
3012
|
+
compression: z3.enum(["gzip", "none"]),
|
|
3013
|
+
size_compressed: z3.number().int().min(0),
|
|
3014
|
+
size_uncompressed: z3.number().int().min(0),
|
|
3015
|
+
sha256: z3.string().regex(/^[0-9a-f]{64}$/, "sha256 must be 64 lowercase hex chars")
|
|
3016
|
+
});
|
|
3017
|
+
var ProducerSchema = z3.object({
|
|
3018
|
+
tool: z3.string().min(1),
|
|
3019
|
+
version: z3.string().min(1),
|
|
3020
|
+
agent_id: z3.string().optional()
|
|
3021
|
+
});
|
|
3022
|
+
var SignerSchema = z3.object({
|
|
3023
|
+
algo: z3.literal("ed25519"),
|
|
3024
|
+
public_key: z3.string().min(1),
|
|
3025
|
+
key_id: z3.string().optional()
|
|
3026
|
+
});
|
|
3027
|
+
var CapsuleHeaderSchema = z3.object({
|
|
3028
|
+
schema: z3.literal("plur.capsule/1"),
|
|
3029
|
+
product_type: z3.enum(["engram-pack", "skill"]),
|
|
3030
|
+
manifest_summary: ManifestSummarySchema,
|
|
3031
|
+
payload: PayloadDescriptorSchema,
|
|
3032
|
+
created_at: z3.string().datetime({ offset: true }),
|
|
3033
|
+
producer: ProducerSchema,
|
|
3034
|
+
signer: SignerSchema.nullable().default(null)
|
|
3035
|
+
});
|
|
3036
|
+
function parseCapsulePreamble(buf) {
|
|
3037
|
+
if (buf.length < PREAMBLE_LEN) {
|
|
3038
|
+
throw new Error(`capsule: truncated preamble (got ${buf.length} bytes, need ${PREAMBLE_LEN})`);
|
|
3039
|
+
}
|
|
3040
|
+
if (buf.compare(CAPSULE_MAGIC, 0, 4, 0, 4) !== 0) {
|
|
3041
|
+
throw new Error(`capsule: bad magic (expected ${CAPSULE_MAGIC_HEX})`);
|
|
3042
|
+
}
|
|
3043
|
+
const formatVersion = buf.readUInt16LE(4);
|
|
3044
|
+
if (!SUPPORTED_FORMAT_VERSIONS.includes(formatVersion)) {
|
|
3045
|
+
throw new Error(`capsule: unsupported FormatVersion 0x${formatVersion.toString(16).padStart(4, "0")}`);
|
|
3046
|
+
}
|
|
3047
|
+
const flags = buf.readUInt16LE(6);
|
|
3048
|
+
if ((flags & CAPSULE_FLAG_RESERVED_MASK) !== 0) {
|
|
3049
|
+
throw new Error(`capsule: reserved flag bits set (flags=0x${flags.toString(16).padStart(4, "0")})`);
|
|
3050
|
+
}
|
|
3051
|
+
const headerLen = buf.readUInt32LE(8);
|
|
3052
|
+
if (headerLen === 0) {
|
|
3053
|
+
throw new Error("capsule: HeaderLen must be > 0");
|
|
3054
|
+
}
|
|
3055
|
+
if (headerLen > CAPSULE_SIZE_LIMITS.HARD_BYTES) {
|
|
3056
|
+
throw new Error(`capsule: HeaderLen ${headerLen} exceeds hard size limit`);
|
|
3057
|
+
}
|
|
3058
|
+
return { formatVersion, flags, headerLen };
|
|
3059
|
+
}
|
|
3060
|
+
function serializeCapsulePreamble(p) {
|
|
3061
|
+
const buf = Buffer.alloc(PREAMBLE_LEN);
|
|
3062
|
+
CAPSULE_MAGIC.copy(buf, 0);
|
|
3063
|
+
buf.writeUInt16LE(p.formatVersion, 4);
|
|
3064
|
+
buf.writeUInt16LE(p.flags, 6);
|
|
3065
|
+
buf.writeUInt32LE(p.headerLen, 8);
|
|
3066
|
+
return buf;
|
|
3067
|
+
}
|
|
3068
|
+
function hasFlag(flags, flag) {
|
|
3069
|
+
return (flags & flag) === flag;
|
|
3070
|
+
}
|
|
3071
|
+
|
|
3072
|
+
// src/capsule.ts
|
|
3073
|
+
import { createHash as createHash2 } from "crypto";
|
|
3074
|
+
function writeCapsule(opts) {
|
|
3075
|
+
const compression = opts.compression ?? "gzip";
|
|
3076
|
+
const productType = opts.productType ?? "engram-pack";
|
|
3077
|
+
const createdAt = opts.createdAt ?? (/* @__PURE__ */ new Date()).toISOString();
|
|
3078
|
+
const signer = opts.signer ?? null;
|
|
3079
|
+
if (signer !== null && (!opts.signature || opts.signature.length !== ED25519_SIG_LEN)) {
|
|
3080
|
+
throw new Error(`writeCapsule: signer set but signature missing or not ${ED25519_SIG_LEN} bytes`);
|
|
3081
|
+
}
|
|
3082
|
+
if (signer === null && opts.signature) {
|
|
3083
|
+
throw new Error("writeCapsule: signature provided without signer \u2014 refuse ambiguous envelope");
|
|
3084
|
+
}
|
|
3085
|
+
const sha256 = createHash2("sha256").update(opts.payload).digest("hex");
|
|
3086
|
+
const sizeCompressed = opts.payload.length;
|
|
3087
|
+
const sizeUncompressed = opts.sizeUncompressed ?? sizeCompressed;
|
|
3088
|
+
const header = CapsuleHeaderSchema.parse({
|
|
3089
|
+
schema: "plur.capsule/1",
|
|
3090
|
+
product_type: productType,
|
|
3091
|
+
manifest_summary: opts.manifestSummary,
|
|
3092
|
+
payload: {
|
|
3093
|
+
compression,
|
|
3094
|
+
size_compressed: sizeCompressed,
|
|
3095
|
+
size_uncompressed: sizeUncompressed,
|
|
3096
|
+
sha256
|
|
3097
|
+
},
|
|
3098
|
+
created_at: createdAt,
|
|
3099
|
+
producer: opts.producer,
|
|
3100
|
+
signer
|
|
3101
|
+
});
|
|
3102
|
+
const headerJson = Buffer.from(JSON.stringify(header), "utf-8");
|
|
3103
|
+
let flags = 0;
|
|
3104
|
+
if (compression === "gzip") flags |= CAPSULE_FLAGS.COMPRESSED;
|
|
3105
|
+
if (signer !== null) flags |= CAPSULE_FLAGS.SIGNED;
|
|
3106
|
+
const preamble = serializeCapsulePreamble({
|
|
3107
|
+
formatVersion: FORMAT_VERSION_V1,
|
|
3108
|
+
flags,
|
|
3109
|
+
headerLen: headerJson.length
|
|
3110
|
+
});
|
|
3111
|
+
const totalLen = PREAMBLE_LEN + headerJson.length + opts.payload.length + (signer !== null ? ED25519_SIG_LEN : 0);
|
|
3112
|
+
if (totalLen > CAPSULE_SIZE_LIMITS.HARD_BYTES) {
|
|
3113
|
+
throw new Error(`writeCapsule: capsule size ${totalLen} exceeds hard limit`);
|
|
3114
|
+
}
|
|
3115
|
+
const parts = [preamble, headerJson, opts.payload];
|
|
3116
|
+
if (signer !== null && opts.signature) parts.push(opts.signature);
|
|
3117
|
+
return Buffer.concat(parts, totalLen);
|
|
3118
|
+
}
|
|
3119
|
+
function readCapsule(buf) {
|
|
3120
|
+
if (buf.length > CAPSULE_SIZE_LIMITS.HARD_BYTES) {
|
|
3121
|
+
throw new Error(`readCapsule: capsule size ${buf.length} exceeds hard limit`);
|
|
3122
|
+
}
|
|
3123
|
+
const preamble = parseCapsulePreamble(buf);
|
|
3124
|
+
const headerStart = PREAMBLE_LEN;
|
|
3125
|
+
const headerEnd = headerStart + preamble.headerLen;
|
|
3126
|
+
if (buf.length < headerEnd) {
|
|
3127
|
+
throw new Error(`readCapsule: truncated header (need ${headerEnd} bytes, got ${buf.length})`);
|
|
3128
|
+
}
|
|
3129
|
+
const headerJson = buf.subarray(headerStart, headerEnd).toString("utf-8");
|
|
3130
|
+
let parsedHeader;
|
|
3131
|
+
try {
|
|
3132
|
+
parsedHeader = JSON.parse(headerJson);
|
|
3133
|
+
} catch (err) {
|
|
3134
|
+
throw new Error(`readCapsule: malformed header JSON \u2014 ${err.message}`);
|
|
3135
|
+
}
|
|
3136
|
+
const header = CapsuleHeaderSchema.parse(parsedHeader);
|
|
3137
|
+
const isSigned = hasFlag(preamble.flags, CAPSULE_FLAGS.SIGNED);
|
|
3138
|
+
const sigLen = isSigned ? ED25519_SIG_LEN : 0;
|
|
3139
|
+
const payloadEnd = buf.length - sigLen;
|
|
3140
|
+
if (payloadEnd < headerEnd) {
|
|
3141
|
+
throw new Error("readCapsule: payload region underflow");
|
|
3142
|
+
}
|
|
3143
|
+
const payload = buf.subarray(headerEnd, payloadEnd);
|
|
3144
|
+
const signature = isSigned ? buf.subarray(payloadEnd) : null;
|
|
3145
|
+
if (payload.length !== header.payload.size_compressed) {
|
|
3146
|
+
throw new Error(
|
|
3147
|
+
`readCapsule: payload size mismatch (header=${header.payload.size_compressed}, actual=${payload.length})`
|
|
3148
|
+
);
|
|
3149
|
+
}
|
|
3150
|
+
const actualSha = createHash2("sha256").update(payload).digest("hex");
|
|
3151
|
+
if (actualSha !== header.payload.sha256) {
|
|
3152
|
+
throw new Error(`readCapsule: integrity mismatch (header=${header.payload.sha256}, actual=${actualSha})`);
|
|
3153
|
+
}
|
|
3154
|
+
const compressionFlagSet = hasFlag(preamble.flags, CAPSULE_FLAGS.COMPRESSED);
|
|
3155
|
+
const headerSaysCompressed = header.payload.compression === "gzip";
|
|
3156
|
+
if (compressionFlagSet !== headerSaysCompressed) {
|
|
3157
|
+
throw new Error(
|
|
3158
|
+
`readCapsule: COMPRESSED flag (${compressionFlagSet}) disagrees with header.payload.compression (${header.payload.compression})`
|
|
3159
|
+
);
|
|
3160
|
+
}
|
|
3161
|
+
return { header, payload: Buffer.from(payload), signature: signature ? Buffer.from(signature) : null };
|
|
3162
|
+
}
|
|
3163
|
+
function verifyCapsuleIntegrity(buf) {
|
|
3164
|
+
try {
|
|
3165
|
+
readCapsule(buf);
|
|
3166
|
+
return true;
|
|
3167
|
+
} catch {
|
|
3168
|
+
return false;
|
|
3169
|
+
}
|
|
3170
|
+
}
|
|
3171
|
+
|
|
2939
3172
|
// src/index.ts
|
|
2940
3173
|
var COMMITMENT_MULTIPLIER = {
|
|
2941
3174
|
locked: 1,
|
|
@@ -2973,6 +3206,9 @@ var Plur = class {
|
|
|
2973
3206
|
if (this.config.index) {
|
|
2974
3207
|
this.indexedStorage = new IndexedStorage(this.paths.engrams, this.paths.db, this.config.stores);
|
|
2975
3208
|
}
|
|
3209
|
+
if (this.config.embeddings?.enabled === false) {
|
|
3210
|
+
setEmbeddingsEnabled(false, "embeddings disabled in config.yaml (embeddings.enabled = false)");
|
|
3211
|
+
}
|
|
2976
3212
|
}
|
|
2977
3213
|
/**
|
|
2978
3214
|
* Load engrams from primary store + all configured stores, with mtime-based caching.
|
|
@@ -3014,17 +3250,17 @@ var Plur = class {
|
|
|
3014
3250
|
return all;
|
|
3015
3251
|
}
|
|
3016
3252
|
/** Load engrams from a path with mtime-based caching */
|
|
3017
|
-
_loadCached(
|
|
3253
|
+
_loadCached(path3) {
|
|
3018
3254
|
let mtime;
|
|
3019
3255
|
try {
|
|
3020
|
-
mtime = fs4.statSync(
|
|
3256
|
+
mtime = fs4.statSync(path3, { bigint: true }).mtimeNs;
|
|
3021
3257
|
} catch {
|
|
3022
3258
|
return [];
|
|
3023
3259
|
}
|
|
3024
|
-
const cached = this._engramCache.get(
|
|
3260
|
+
const cached = this._engramCache.get(path3);
|
|
3025
3261
|
if (cached && cached.mtime === mtime) return cached.engrams;
|
|
3026
|
-
const engrams = loadEngrams(
|
|
3027
|
-
this._engramCache.set(
|
|
3262
|
+
const engrams = loadEngrams(path3);
|
|
3263
|
+
this._engramCache.set(path3, { mtime, engrams });
|
|
3028
3264
|
return engrams;
|
|
3029
3265
|
}
|
|
3030
3266
|
/**
|
|
@@ -3038,9 +3274,9 @@ var Plur = class {
|
|
|
3038
3274
|
* invalidation on write removes the filesystem as a source of cache
|
|
3039
3275
|
* freshness and closes the race. See issue #25.
|
|
3040
3276
|
*/
|
|
3041
|
-
_writeEngrams(
|
|
3042
|
-
saveEngrams(
|
|
3043
|
-
this._engramCache.delete(
|
|
3277
|
+
_writeEngrams(path3, engrams) {
|
|
3278
|
+
saveEngrams(path3, engrams);
|
|
3279
|
+
this._engramCache.delete(path3);
|
|
3044
3280
|
}
|
|
3045
3281
|
/** Find which store owns an engram by ID. For namespaced IDs, strips prefix to find in store. */
|
|
3046
3282
|
_findEngramStore(id) {
|
|
@@ -3168,7 +3404,8 @@ var Plur = class {
|
|
|
3168
3404
|
narrower: [],
|
|
3169
3405
|
related: [],
|
|
3170
3406
|
conflicts: conflictIds
|
|
3171
|
-
} : void 0
|
|
3407
|
+
} : void 0,
|
|
3408
|
+
pinned: context?.pinned === true ? true : void 0
|
|
3172
3409
|
};
|
|
3173
3410
|
engrams.push(engram);
|
|
3174
3411
|
this._writeEngrams(this.paths.engrams, engrams);
|
|
@@ -3201,12 +3438,12 @@ var Plur = class {
|
|
|
3201
3438
|
}
|
|
3202
3439
|
/** Async learn with LLM-driven deduplication (Ideas 1+2+19). */
|
|
3203
3440
|
async learnAsync(statement, context) {
|
|
3204
|
-
const { learnAsync: learnAsyncImpl } = await import("./learn-async-
|
|
3441
|
+
const { learnAsync: learnAsyncImpl } = await import("./learn-async-IISWD7HC.js");
|
|
3205
3442
|
return learnAsyncImpl(this._learnAsyncDeps(), statement, context);
|
|
3206
3443
|
}
|
|
3207
3444
|
/** Batch learn with LLM dedup. */
|
|
3208
3445
|
async learnBatch(statements, llm) {
|
|
3209
|
-
const { learnBatch: learnBatchImpl } = await import("./learn-async-
|
|
3446
|
+
const { learnBatch: learnBatchImpl } = await import("./learn-async-IISWD7HC.js");
|
|
3210
3447
|
return learnBatchImpl(this._learnAsyncDeps(), statements, llm);
|
|
3211
3448
|
}
|
|
3212
3449
|
/**
|
|
@@ -3247,6 +3484,26 @@ var Plur = class {
|
|
|
3247
3484
|
this._reactivateResults(results);
|
|
3248
3485
|
return results;
|
|
3249
3486
|
}
|
|
3487
|
+
/**
|
|
3488
|
+
* Hybrid search with diagnostic metadata — returns both the engrams and
|
|
3489
|
+
* whether embeddings actually contributed (mode: "hybrid" vs "hybrid-degraded").
|
|
3490
|
+
* Use this when you want to surface degraded-mode warnings to users.
|
|
3491
|
+
*/
|
|
3492
|
+
async recallHybridWithMeta(query, options) {
|
|
3493
|
+
const filtered = this._filterEngrams(options);
|
|
3494
|
+
const limit = options?.limit ?? 20;
|
|
3495
|
+
const result = await hybridSearchWithMeta(filtered, query, limit, this.paths.root);
|
|
3496
|
+
this._reactivateResults(result.engrams);
|
|
3497
|
+
return result;
|
|
3498
|
+
}
|
|
3499
|
+
/** Inspect embedder availability without forcing a load. */
|
|
3500
|
+
embedderStatus() {
|
|
3501
|
+
return embedderStatus();
|
|
3502
|
+
}
|
|
3503
|
+
/** Reset cached embedder failure state — next call will retry the model load. */
|
|
3504
|
+
resetEmbedder() {
|
|
3505
|
+
resetEmbedder();
|
|
3506
|
+
}
|
|
3250
3507
|
/** Embedding search returning {engram, score}[] with cosine similarity scores. Async, no API calls. */
|
|
3251
3508
|
async similaritySearch(query, options) {
|
|
3252
3509
|
const filtered = this._filterEngrams(options);
|
|
@@ -3375,11 +3632,16 @@ var Plur = class {
|
|
|
3375
3632
|
let embeddingBoosts;
|
|
3376
3633
|
try {
|
|
3377
3634
|
const engrams = this._loadAllEngrams().filter((e) => e.status === "active");
|
|
3378
|
-
const results = await
|
|
3635
|
+
const results = await embeddingSearchWithScores(
|
|
3636
|
+
engrams,
|
|
3637
|
+
task,
|
|
3638
|
+
engrams.length,
|
|
3639
|
+
this.paths.root
|
|
3640
|
+
);
|
|
3379
3641
|
if (results.length > 0) {
|
|
3380
3642
|
embeddingBoosts = /* @__PURE__ */ new Map();
|
|
3381
|
-
for (
|
|
3382
|
-
embeddingBoosts.set(
|
|
3643
|
+
for (const r of results) {
|
|
3644
|
+
embeddingBoosts.set(r.engram.id, r.score);
|
|
3383
3645
|
}
|
|
3384
3646
|
}
|
|
3385
3647
|
} catch {
|
|
@@ -3510,6 +3772,28 @@ var Plur = class {
|
|
|
3510
3772
|
return true;
|
|
3511
3773
|
});
|
|
3512
3774
|
}
|
|
3775
|
+
/**
|
|
3776
|
+
* Toggle the always-load (pinned) flag for an engram.
|
|
3777
|
+
* Returns the updated engram on success, null if not found.
|
|
3778
|
+
*/
|
|
3779
|
+
setPinned(id, pinned) {
|
|
3780
|
+
return withLock(this.paths.engrams, () => {
|
|
3781
|
+
const engrams = loadEngrams(this.paths.engrams);
|
|
3782
|
+
const idx = engrams.findIndex((e2) => e2.id === id);
|
|
3783
|
+
if (idx === -1) return null;
|
|
3784
|
+
const e = engrams[idx];
|
|
3785
|
+
const updated = { ...e, pinned: pinned === true ? true : void 0 };
|
|
3786
|
+
engrams[idx] = updated;
|
|
3787
|
+
this._writeEngrams(this.paths.engrams, engrams);
|
|
3788
|
+
this._syncIndex();
|
|
3789
|
+
return updated;
|
|
3790
|
+
});
|
|
3791
|
+
}
|
|
3792
|
+
/** List engrams that have pinned: true. */
|
|
3793
|
+
listPinned() {
|
|
3794
|
+
const all = this._loadAllEngrams();
|
|
3795
|
+
return all.filter((e) => e.pinned === true && e.status === "active");
|
|
3796
|
+
}
|
|
3513
3797
|
/** Set engram status to 'retired'. Supports primary and store engrams. */
|
|
3514
3798
|
forget(id, reason) {
|
|
3515
3799
|
const foundInPrimary = withLock(this.paths.engrams, () => {
|
|
@@ -3875,20 +4159,19 @@ Generate an improved version of the procedure that prevents this failure. Return
|
|
|
3875
4159
|
autoDiscoverStores(cwd) {
|
|
3876
4160
|
const startDir = cwd || process.cwd();
|
|
3877
4161
|
const discovered = [];
|
|
3878
|
-
const tmpDir =
|
|
4162
|
+
const tmpDir = tmpdir();
|
|
3879
4163
|
if (this.paths.root.startsWith(tmpDir) || this.paths.root.startsWith("/tmp/")) {
|
|
3880
4164
|
return discovered;
|
|
3881
4165
|
}
|
|
3882
4166
|
const knownPaths = new Set((this.config.stores ?? []).map((s) => s.path));
|
|
3883
|
-
const primaryDir =
|
|
4167
|
+
const primaryDir = dirname2(this.paths.engrams);
|
|
3884
4168
|
let dir = startDir;
|
|
3885
|
-
const { join: join5, dirname: dirname3, basename: basename2 } = path3;
|
|
3886
4169
|
const visited = /* @__PURE__ */ new Set();
|
|
3887
4170
|
while (dir && !visited.has(dir)) {
|
|
3888
4171
|
visited.add(dir);
|
|
3889
4172
|
const candidate = join5(dir, ".plur", "engrams.yaml");
|
|
3890
4173
|
if (join5(dir, ".plur") === primaryDir) {
|
|
3891
|
-
dir =
|
|
4174
|
+
dir = dirname2(dir);
|
|
3892
4175
|
continue;
|
|
3893
4176
|
}
|
|
3894
4177
|
if (fs4.existsSync(candidate) && !knownPaths.has(candidate)) {
|
|
@@ -3907,7 +4190,7 @@ Generate an improved version of the procedure that prevents this failure. Return
|
|
|
3907
4190
|
logger.info(`Auto-discovered project store: ${candidate} (${scope})`);
|
|
3908
4191
|
}
|
|
3909
4192
|
if (fs4.existsSync(join5(dir, ".git"))) break;
|
|
3910
|
-
const parent =
|
|
4193
|
+
const parent = dirname2(dir);
|
|
3911
4194
|
if (parent === dir) break;
|
|
3912
4195
|
dir = parent;
|
|
3913
4196
|
}
|
|
@@ -3936,18 +4219,32 @@ Generate an improved version of the procedure that prevents this failure. Return
|
|
|
3936
4219
|
};
|
|
3937
4220
|
export {
|
|
3938
4221
|
ALL_MIGRATIONS,
|
|
4222
|
+
CAPSULE_FLAGS,
|
|
4223
|
+
CAPSULE_FLAG_RESERVED_MASK,
|
|
4224
|
+
CAPSULE_MAGIC,
|
|
4225
|
+
CAPSULE_MAGIC_HEX,
|
|
4226
|
+
CAPSULE_SIZE_LIMITS,
|
|
3939
4227
|
COMMITMENT_MULTIPLIER,
|
|
3940
4228
|
CURRENT_SCHEMA_VERSION,
|
|
4229
|
+
CapsuleHeaderSchema,
|
|
3941
4230
|
DomainCoverageSchema,
|
|
4231
|
+
ED25519_SIG_LEN,
|
|
3942
4232
|
EvidenceEntrySchema,
|
|
4233
|
+
FORMAT_VERSION_V1,
|
|
3943
4234
|
FalsificationSchema,
|
|
3944
4235
|
HierarchyPositionSchema,
|
|
3945
4236
|
IndexedStorage,
|
|
4237
|
+
ManifestSummarySchema,
|
|
3946
4238
|
MetaConfidenceSchema,
|
|
3947
4239
|
MetaFieldSchema,
|
|
3948
4240
|
PLATITUDE_PATTERNS,
|
|
4241
|
+
PREAMBLE_LEN,
|
|
4242
|
+
PayloadDescriptorSchema,
|
|
3949
4243
|
Plur,
|
|
4244
|
+
ProducerSchema,
|
|
4245
|
+
SUPPORTED_FORMAT_VERSIONS,
|
|
3950
4246
|
SessionBreadcrumbs,
|
|
4247
|
+
SignerSchema,
|
|
3951
4248
|
SqliteStore,
|
|
3952
4249
|
StructuralTemplateSchema,
|
|
3953
4250
|
YamlStore,
|
|
@@ -3986,6 +4283,7 @@ export {
|
|
|
3986
4283
|
getCachedUpdateCheck,
|
|
3987
4284
|
getProfileForInjection,
|
|
3988
4285
|
getSchemaVersion,
|
|
4286
|
+
hasFlag,
|
|
3989
4287
|
isPlatitude,
|
|
3990
4288
|
listHistoryMonths,
|
|
3991
4289
|
loadProfileCache,
|
|
@@ -3994,8 +4292,10 @@ export {
|
|
|
3994
4292
|
needsSummary,
|
|
3995
4293
|
normalizeStatement,
|
|
3996
4294
|
organizeHierarchy,
|
|
4295
|
+
parseCapsulePreamble,
|
|
3997
4296
|
parseDedupResponse,
|
|
3998
4297
|
profileNeedsRegeneration,
|
|
4298
|
+
readCapsule,
|
|
3999
4299
|
readHistory,
|
|
4000
4300
|
readHistoryForEngram,
|
|
4001
4301
|
recallAuto,
|
|
@@ -4005,9 +4305,12 @@ export {
|
|
|
4005
4305
|
saveProfileCache,
|
|
4006
4306
|
selectModel,
|
|
4007
4307
|
selectModelForOperation,
|
|
4308
|
+
serializeCapsulePreamble,
|
|
4008
4309
|
setSchemaVersion,
|
|
4009
4310
|
strengthToStatus,
|
|
4010
4311
|
tokenSimilarity,
|
|
4011
4312
|
validateMetaEngram,
|
|
4012
|
-
|
|
4313
|
+
verifyCapsuleIntegrity,
|
|
4314
|
+
withAsyncLock,
|
|
4315
|
+
writeCapsule
|
|
4013
4316
|
};
|
|
@@ -6,11 +6,10 @@ import {
|
|
|
6
6
|
logger,
|
|
7
7
|
parseDedupResponse,
|
|
8
8
|
saveEngrams
|
|
9
|
-
} from "./chunk-
|
|
9
|
+
} from "./chunk-EQPTF4JZ.js";
|
|
10
10
|
import {
|
|
11
11
|
withLock
|
|
12
12
|
} from "./chunk-PRK3B7WR.js";
|
|
13
|
-
import "./chunk-2ZDO52B4.js";
|
|
14
13
|
|
|
15
14
|
// src/learn-async.ts
|
|
16
15
|
function executeDedupDecision(deps, statement, context, decision, targetId, conflicts) {
|