@xdarkicex/openclaw-memory-libravdb 1.4.66 → 1.4.68
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/context-engine.js +24 -18
- package/dist/index.js +202 -45
- package/dist/markdown-ingest.js +180 -19
- package/dist/types.d.ts +2 -0
- package/docs/configuration.md +4 -0
- package/openclaw.plugin.json +7 -1
- package/package.json +1 -1
package/dist/context-engine.js
CHANGED
|
@@ -605,11 +605,12 @@ export function buildContextEngineFactory(runtime, cfg, logger = console) {
|
|
|
605
605
|
info: { id: "libravdb-memory", name: "LibraVDB Memory", ownsCompaction: true },
|
|
606
606
|
ownsCompaction: true,
|
|
607
607
|
async bootstrap(args) {
|
|
608
|
+
const sessionId = requireSessionId(args.sessionId, "bootstrap");
|
|
608
609
|
const userId = resolveUserId({
|
|
609
610
|
userIdOverride: args.userId,
|
|
610
611
|
sessionKey: args.sessionKey,
|
|
611
612
|
});
|
|
612
|
-
logger.info?.(`LibraVDB bootstrap sessionId=${
|
|
613
|
+
logger.info?.(`LibraVDB bootstrap sessionId=${sessionId} userId=${userId} ` +
|
|
613
614
|
`sessionKey=${args.sessionKey ?? "(none)"}`);
|
|
614
615
|
const kernel = await getKernelOrNull("bootstrap");
|
|
615
616
|
if (kernel) {
|
|
@@ -623,7 +624,7 @@ export function buildContextEngineFactory(runtime, cfg, logger = console) {
|
|
|
623
624
|
// Proceed even if initialize session fails or doesn't return nonce if secret optional
|
|
624
625
|
}
|
|
625
626
|
return await kernel.bootstrapSession({
|
|
626
|
-
sessionId
|
|
627
|
+
sessionId,
|
|
627
628
|
sessionKey: args.sessionKey,
|
|
628
629
|
userId,
|
|
629
630
|
});
|
|
@@ -631,23 +632,25 @@ export function buildContextEngineFactory(runtime, cfg, logger = console) {
|
|
|
631
632
|
const rpc = await runtime.getRpc();
|
|
632
633
|
return await rpc.call("bootstrap_session_kernel", {
|
|
633
634
|
...args,
|
|
635
|
+
sessionId,
|
|
634
636
|
userId,
|
|
635
637
|
});
|
|
636
638
|
},
|
|
637
639
|
async ingest(args) {
|
|
640
|
+
const sessionId = requireSessionId(args.sessionId, "ingest");
|
|
638
641
|
const userId = resolveUserId({
|
|
639
642
|
userIdOverride: args.userId,
|
|
640
643
|
sessionKey: args.sessionKey,
|
|
641
644
|
});
|
|
642
645
|
const message = normalizeKernelMessage(args.message);
|
|
643
|
-
logger.info?.(`LibraVDB ingest sessionId=${
|
|
646
|
+
logger.info?.(`LibraVDB ingest sessionId=${sessionId} userId=${userId} ` +
|
|
644
647
|
`role=${message.role} heartbeat=${args.isHeartbeat ?? false} ` +
|
|
645
648
|
`contentLen=${message.content.length}`);
|
|
646
649
|
try {
|
|
647
650
|
const kernel = await getKernelOrNull("ingest");
|
|
648
651
|
if (kernel) {
|
|
649
652
|
return await kernel.ingestMessage({
|
|
650
|
-
sessionId
|
|
653
|
+
sessionId,
|
|
651
654
|
sessionKey: args.sessionKey,
|
|
652
655
|
userId,
|
|
653
656
|
message,
|
|
@@ -657,17 +660,19 @@ export function buildContextEngineFactory(runtime, cfg, logger = console) {
|
|
|
657
660
|
const rpc = await runtime.getRpc();
|
|
658
661
|
return await rpc.call("ingest_message_kernel", {
|
|
659
662
|
...args,
|
|
663
|
+
sessionId,
|
|
660
664
|
userId,
|
|
661
665
|
message,
|
|
662
666
|
});
|
|
663
667
|
}
|
|
664
668
|
catch (error) {
|
|
665
|
-
logger.warn?.(`LibraVDB ingest failed sessionId=${
|
|
669
|
+
logger.warn?.(`LibraVDB ingest failed sessionId=${sessionId}: ` +
|
|
666
670
|
`${error instanceof Error ? error.message : String(error)}`);
|
|
667
671
|
throw error;
|
|
668
672
|
}
|
|
669
673
|
},
|
|
670
674
|
async assemble(args) {
|
|
675
|
+
const sessionId = requireSessionId(args.sessionId, "assemble");
|
|
671
676
|
const userId = resolveUserId({
|
|
672
677
|
userIdOverride: args.userId,
|
|
673
678
|
sessionKey: args.sessionKey,
|
|
@@ -687,14 +692,14 @@ export function buildContextEngineFactory(runtime, cfg, logger = console) {
|
|
|
687
692
|
logPredictiveCompactionAttempt({
|
|
688
693
|
logger,
|
|
689
694
|
phase: "assemble",
|
|
690
|
-
sessionId
|
|
695
|
+
sessionId,
|
|
691
696
|
currentTokenCount: currentContextTokens,
|
|
692
697
|
threshold: dynamicCompactThreshold,
|
|
693
698
|
targetSize: predictiveTargetSize,
|
|
694
699
|
tokenBudget: args.tokenBudget,
|
|
695
700
|
});
|
|
696
701
|
const compactionResult = await runCompaction({
|
|
697
|
-
sessionId
|
|
702
|
+
sessionId,
|
|
698
703
|
targetSize: predictiveTargetSize,
|
|
699
704
|
tokenBudget: args.tokenBudget,
|
|
700
705
|
force: true,
|
|
@@ -703,7 +708,7 @@ export function buildContextEngineFactory(runtime, cfg, logger = console) {
|
|
|
703
708
|
logPredictiveCompactionOutcome({
|
|
704
709
|
logger,
|
|
705
710
|
phase: "assemble",
|
|
706
|
-
sessionId
|
|
711
|
+
sessionId,
|
|
707
712
|
currentTokenCount: currentContextTokens,
|
|
708
713
|
threshold: dynamicCompactThreshold,
|
|
709
714
|
targetSize: predictiveTargetSize,
|
|
@@ -721,7 +726,7 @@ export function buildContextEngineFactory(runtime, cfg, logger = console) {
|
|
|
721
726
|
if (kernel) {
|
|
722
727
|
try {
|
|
723
728
|
const assembled = normalizeAssembleResult(await kernel.assembleContext({
|
|
724
|
-
sessionId
|
|
729
|
+
sessionId,
|
|
725
730
|
sessionKey: args.sessionKey,
|
|
726
731
|
userId,
|
|
727
732
|
queryText: args.prompt ?? "",
|
|
@@ -733,7 +738,7 @@ export function buildContextEngineFactory(runtime, cfg, logger = console) {
|
|
|
733
738
|
return enforceTokenBudgetInvariant(await augmentWithExactRecall(assembled, {
|
|
734
739
|
queryText: args.prompt ?? messages[messages.length - 1]?.content ?? "",
|
|
735
740
|
userId,
|
|
736
|
-
sessionId
|
|
741
|
+
sessionId,
|
|
737
742
|
tokenBudget: args.tokenBudget,
|
|
738
743
|
}), args.tokenBudget);
|
|
739
744
|
}
|
|
@@ -745,7 +750,7 @@ export function buildContextEngineFactory(runtime, cfg, logger = console) {
|
|
|
745
750
|
const rpc = await runtime.getRpc();
|
|
746
751
|
try {
|
|
747
752
|
const resp = await rpc.call("assemble_context_internal", {
|
|
748
|
-
sessionId
|
|
753
|
+
sessionId,
|
|
749
754
|
sessionKey: args.sessionKey,
|
|
750
755
|
userId,
|
|
751
756
|
messages,
|
|
@@ -758,7 +763,7 @@ export function buildContextEngineFactory(runtime, cfg, logger = console) {
|
|
|
758
763
|
return enforceTokenBudgetInvariant(await augmentWithExactRecall(assembled, {
|
|
759
764
|
queryText: args.prompt ?? messages[messages.length - 1]?.content ?? "",
|
|
760
765
|
userId,
|
|
761
|
-
sessionId
|
|
766
|
+
sessionId,
|
|
762
767
|
tokenBudget: args.tokenBudget,
|
|
763
768
|
}), args.tokenBudget);
|
|
764
769
|
}
|
|
@@ -771,6 +776,7 @@ export function buildContextEngineFactory(runtime, cfg, logger = console) {
|
|
|
771
776
|
return await runCompaction(args);
|
|
772
777
|
},
|
|
773
778
|
async afterTurn(args) {
|
|
779
|
+
const sessionId = requireSessionId(args.sessionId, "afterTurn");
|
|
774
780
|
const userId = resolveUserId({
|
|
775
781
|
userIdOverride: args.userId,
|
|
776
782
|
sessionKey: args.sessionKey,
|
|
@@ -778,7 +784,7 @@ export function buildContextEngineFactory(runtime, cfg, logger = console) {
|
|
|
778
784
|
const afterTurnMessages = selectAfterTurnMessages(args.messages, args.prePromptMessageCount, logger);
|
|
779
785
|
const messages = normalizeKernelMessages(afterTurnMessages);
|
|
780
786
|
const msgCount = messages.length;
|
|
781
|
-
logger.info?.(`LibraVDB afterTurn sessionId=${
|
|
787
|
+
logger.info?.(`LibraVDB afterTurn sessionId=${sessionId} userId=${userId} ` +
|
|
782
788
|
`messageCount=${msgCount} totalMessages=${args.messages.length} ` +
|
|
783
789
|
`prePromptMessageCount=${args.prePromptMessageCount ?? "unknown"} ` +
|
|
784
790
|
`heartbeat=${args.isHeartbeat ?? false}`);
|
|
@@ -789,14 +795,14 @@ export function buildContextEngineFactory(runtime, cfg, logger = console) {
|
|
|
789
795
|
: undefined);
|
|
790
796
|
if (kernel) {
|
|
791
797
|
const result = await kernel.afterTurn({
|
|
792
|
-
sessionId
|
|
798
|
+
sessionId,
|
|
793
799
|
sessionKey: args.sessionKey,
|
|
794
800
|
userId,
|
|
795
801
|
messages,
|
|
796
802
|
isHeartbeat: args.isHeartbeat,
|
|
797
803
|
});
|
|
798
804
|
await performAfterTurnPredictiveCompaction({
|
|
799
|
-
sessionId
|
|
805
|
+
sessionId,
|
|
800
806
|
tokenBudget: args.tokenBudget,
|
|
801
807
|
currentTokenCount,
|
|
802
808
|
});
|
|
@@ -804,21 +810,21 @@ export function buildContextEngineFactory(runtime, cfg, logger = console) {
|
|
|
804
810
|
}
|
|
805
811
|
const rpc = await runtime.getRpc();
|
|
806
812
|
const result = await rpc.call("after_turn_kernel", {
|
|
807
|
-
sessionId
|
|
813
|
+
sessionId,
|
|
808
814
|
sessionKey: args.sessionKey,
|
|
809
815
|
userId,
|
|
810
816
|
messages,
|
|
811
817
|
isHeartbeat: args.isHeartbeat,
|
|
812
818
|
});
|
|
813
819
|
await performAfterTurnPredictiveCompaction({
|
|
814
|
-
sessionId
|
|
820
|
+
sessionId,
|
|
815
821
|
tokenBudget: args.tokenBudget,
|
|
816
822
|
currentTokenCount,
|
|
817
823
|
});
|
|
818
824
|
return result;
|
|
819
825
|
}
|
|
820
826
|
catch (error) {
|
|
821
|
-
logger.warn?.(`LibraVDB afterTurn failed sessionId=${
|
|
827
|
+
logger.warn?.(`LibraVDB afterTurn failed sessionId=${sessionId}: ` +
|
|
822
828
|
`${error instanceof Error ? error.message : String(error)}`);
|
|
823
829
|
throw error;
|
|
824
830
|
}
|
package/dist/index.js
CHANGED
|
@@ -9467,7 +9467,7 @@ var require_service_config = __commonJS({
|
|
|
9467
9467
|
exports2.validateRetryThrottling = validateRetryThrottling;
|
|
9468
9468
|
exports2.validateServiceConfig = validateServiceConfig;
|
|
9469
9469
|
exports2.extractAndSelectServiceConfig = extractAndSelectServiceConfig;
|
|
9470
|
-
var
|
|
9470
|
+
var os4 = __require("os");
|
|
9471
9471
|
var constants_1 = require_constants();
|
|
9472
9472
|
var DURATION_REGEX = /^\d+(\.\d{1,9})?s$/;
|
|
9473
9473
|
var CLIENT_LANGUAGE_STRING = "node";
|
|
@@ -9766,7 +9766,7 @@ var require_service_config = __commonJS({
|
|
|
9766
9766
|
if (Array.isArray(validatedConfig.clientHostname)) {
|
|
9767
9767
|
let hostnameMatched = false;
|
|
9768
9768
|
for (const hostname2 of validatedConfig.clientHostname) {
|
|
9769
|
-
if (hostname2 ===
|
|
9769
|
+
if (hostname2 === os4.hostname()) {
|
|
9770
9770
|
hostnameMatched = true;
|
|
9771
9771
|
}
|
|
9772
9772
|
}
|
|
@@ -24066,7 +24066,7 @@ var require_subchannel_call = __commonJS({
|
|
|
24066
24066
|
Object.defineProperty(exports2, "__esModule", { value: true });
|
|
24067
24067
|
exports2.Http2SubchannelCall = void 0;
|
|
24068
24068
|
var http2 = __require("http2");
|
|
24069
|
-
var
|
|
24069
|
+
var os4 = __require("os");
|
|
24070
24070
|
var constants_1 = require_constants();
|
|
24071
24071
|
var metadata_1 = require_metadata();
|
|
24072
24072
|
var stream_decoder_1 = require_stream_decoder();
|
|
@@ -24074,7 +24074,7 @@ var require_subchannel_call = __commonJS({
|
|
|
24074
24074
|
var constants_2 = require_constants();
|
|
24075
24075
|
var TRACER_NAME = "subchannel_call";
|
|
24076
24076
|
function getSystemErrorName(errno) {
|
|
24077
|
-
for (const [name, num] of Object.entries(
|
|
24077
|
+
for (const [name, num] of Object.entries(os4.constants.errno)) {
|
|
24078
24078
|
if (num === errno) {
|
|
24079
24079
|
return name;
|
|
24080
24080
|
}
|
|
@@ -34223,12 +34223,13 @@ function buildContextEngineFactory(runtime, cfg, logger = console) {
|
|
|
34223
34223
|
info: { id: "libravdb-memory", name: "LibraVDB Memory", ownsCompaction: true },
|
|
34224
34224
|
ownsCompaction: true,
|
|
34225
34225
|
async bootstrap(args) {
|
|
34226
|
+
const sessionId = requireSessionId(args.sessionId, "bootstrap");
|
|
34226
34227
|
const userId = resolveUserId({
|
|
34227
34228
|
userIdOverride: args.userId,
|
|
34228
34229
|
sessionKey: args.sessionKey
|
|
34229
34230
|
});
|
|
34230
34231
|
logger.info?.(
|
|
34231
|
-
`LibraVDB bootstrap sessionId=${
|
|
34232
|
+
`LibraVDB bootstrap sessionId=${sessionId} userId=${userId} sessionKey=${args.sessionKey ?? "(none)"}`
|
|
34232
34233
|
);
|
|
34233
34234
|
const kernel = await getKernelOrNull("bootstrap");
|
|
34234
34235
|
if (kernel) {
|
|
@@ -34240,7 +34241,7 @@ function buildContextEngineFactory(runtime, cfg, logger = console) {
|
|
|
34240
34241
|
} catch (error) {
|
|
34241
34242
|
}
|
|
34242
34243
|
return await kernel.bootstrapSession({
|
|
34243
|
-
sessionId
|
|
34244
|
+
sessionId,
|
|
34244
34245
|
sessionKey: args.sessionKey,
|
|
34245
34246
|
userId
|
|
34246
34247
|
});
|
|
@@ -34248,23 +34249,25 @@ function buildContextEngineFactory(runtime, cfg, logger = console) {
|
|
|
34248
34249
|
const rpc = await runtime.getRpc();
|
|
34249
34250
|
return await rpc.call("bootstrap_session_kernel", {
|
|
34250
34251
|
...args,
|
|
34252
|
+
sessionId,
|
|
34251
34253
|
userId
|
|
34252
34254
|
});
|
|
34253
34255
|
},
|
|
34254
34256
|
async ingest(args) {
|
|
34257
|
+
const sessionId = requireSessionId(args.sessionId, "ingest");
|
|
34255
34258
|
const userId = resolveUserId({
|
|
34256
34259
|
userIdOverride: args.userId,
|
|
34257
34260
|
sessionKey: args.sessionKey
|
|
34258
34261
|
});
|
|
34259
34262
|
const message = normalizeKernelMessage(args.message);
|
|
34260
34263
|
logger.info?.(
|
|
34261
|
-
`LibraVDB ingest sessionId=${
|
|
34264
|
+
`LibraVDB ingest sessionId=${sessionId} userId=${userId} role=${message.role} heartbeat=${args.isHeartbeat ?? false} contentLen=${message.content.length}`
|
|
34262
34265
|
);
|
|
34263
34266
|
try {
|
|
34264
34267
|
const kernel = await getKernelOrNull("ingest");
|
|
34265
34268
|
if (kernel) {
|
|
34266
34269
|
return await kernel.ingestMessage({
|
|
34267
|
-
sessionId
|
|
34270
|
+
sessionId,
|
|
34268
34271
|
sessionKey: args.sessionKey,
|
|
34269
34272
|
userId,
|
|
34270
34273
|
message,
|
|
@@ -34274,17 +34277,19 @@ function buildContextEngineFactory(runtime, cfg, logger = console) {
|
|
|
34274
34277
|
const rpc = await runtime.getRpc();
|
|
34275
34278
|
return await rpc.call("ingest_message_kernel", {
|
|
34276
34279
|
...args,
|
|
34280
|
+
sessionId,
|
|
34277
34281
|
userId,
|
|
34278
34282
|
message
|
|
34279
34283
|
});
|
|
34280
34284
|
} catch (error) {
|
|
34281
34285
|
logger.warn?.(
|
|
34282
|
-
`LibraVDB ingest failed sessionId=${
|
|
34286
|
+
`LibraVDB ingest failed sessionId=${sessionId}: ${error instanceof Error ? error.message : String(error)}`
|
|
34283
34287
|
);
|
|
34284
34288
|
throw error;
|
|
34285
34289
|
}
|
|
34286
34290
|
},
|
|
34287
34291
|
async assemble(args) {
|
|
34292
|
+
const sessionId = requireSessionId(args.sessionId, "assemble");
|
|
34288
34293
|
const userId = resolveUserId({
|
|
34289
34294
|
userIdOverride: args.userId,
|
|
34290
34295
|
sessionKey: args.sessionKey
|
|
@@ -34304,14 +34309,14 @@ function buildContextEngineFactory(runtime, cfg, logger = console) {
|
|
|
34304
34309
|
logPredictiveCompactionAttempt({
|
|
34305
34310
|
logger,
|
|
34306
34311
|
phase: "assemble",
|
|
34307
|
-
sessionId
|
|
34312
|
+
sessionId,
|
|
34308
34313
|
currentTokenCount: currentContextTokens,
|
|
34309
34314
|
threshold: dynamicCompactThreshold,
|
|
34310
34315
|
targetSize: predictiveTargetSize,
|
|
34311
34316
|
tokenBudget: args.tokenBudget
|
|
34312
34317
|
});
|
|
34313
34318
|
const compactionResult = await runCompaction({
|
|
34314
|
-
sessionId
|
|
34319
|
+
sessionId,
|
|
34315
34320
|
targetSize: predictiveTargetSize,
|
|
34316
34321
|
tokenBudget: args.tokenBudget,
|
|
34317
34322
|
force: true,
|
|
@@ -34320,7 +34325,7 @@ function buildContextEngineFactory(runtime, cfg, logger = console) {
|
|
|
34320
34325
|
logPredictiveCompactionOutcome({
|
|
34321
34326
|
logger,
|
|
34322
34327
|
phase: "assemble",
|
|
34323
|
-
sessionId
|
|
34328
|
+
sessionId,
|
|
34324
34329
|
currentTokenCount: currentContextTokens,
|
|
34325
34330
|
threshold: dynamicCompactThreshold,
|
|
34326
34331
|
targetSize: predictiveTargetSize,
|
|
@@ -34339,7 +34344,7 @@ function buildContextEngineFactory(runtime, cfg, logger = console) {
|
|
|
34339
34344
|
if (kernel) {
|
|
34340
34345
|
try {
|
|
34341
34346
|
const assembled = normalizeAssembleResult(await kernel.assembleContext({
|
|
34342
|
-
sessionId
|
|
34347
|
+
sessionId,
|
|
34343
34348
|
sessionKey: args.sessionKey,
|
|
34344
34349
|
userId,
|
|
34345
34350
|
queryText: args.prompt ?? "",
|
|
@@ -34352,7 +34357,7 @@ function buildContextEngineFactory(runtime, cfg, logger = console) {
|
|
|
34352
34357
|
await augmentWithExactRecall(assembled, {
|
|
34353
34358
|
queryText: args.prompt ?? messages[messages.length - 1]?.content ?? "",
|
|
34354
34359
|
userId,
|
|
34355
|
-
sessionId
|
|
34360
|
+
sessionId,
|
|
34356
34361
|
tokenBudget: args.tokenBudget
|
|
34357
34362
|
}),
|
|
34358
34363
|
args.tokenBudget
|
|
@@ -34367,7 +34372,7 @@ function buildContextEngineFactory(runtime, cfg, logger = console) {
|
|
|
34367
34372
|
const rpc = await runtime.getRpc();
|
|
34368
34373
|
try {
|
|
34369
34374
|
const resp = await rpc.call("assemble_context_internal", {
|
|
34370
|
-
sessionId
|
|
34375
|
+
sessionId,
|
|
34371
34376
|
sessionKey: args.sessionKey,
|
|
34372
34377
|
userId,
|
|
34373
34378
|
messages,
|
|
@@ -34381,7 +34386,7 @@ function buildContextEngineFactory(runtime, cfg, logger = console) {
|
|
|
34381
34386
|
await augmentWithExactRecall(assembled, {
|
|
34382
34387
|
queryText: args.prompt ?? messages[messages.length - 1]?.content ?? "",
|
|
34383
34388
|
userId,
|
|
34384
|
-
sessionId
|
|
34389
|
+
sessionId,
|
|
34385
34390
|
tokenBudget: args.tokenBudget
|
|
34386
34391
|
}),
|
|
34387
34392
|
args.tokenBudget
|
|
@@ -34397,6 +34402,7 @@ function buildContextEngineFactory(runtime, cfg, logger = console) {
|
|
|
34397
34402
|
return await runCompaction(args);
|
|
34398
34403
|
},
|
|
34399
34404
|
async afterTurn(args) {
|
|
34405
|
+
const sessionId = requireSessionId(args.sessionId, "afterTurn");
|
|
34400
34406
|
const userId = resolveUserId({
|
|
34401
34407
|
userIdOverride: args.userId,
|
|
34402
34408
|
sessionKey: args.sessionKey
|
|
@@ -34405,7 +34411,7 @@ function buildContextEngineFactory(runtime, cfg, logger = console) {
|
|
|
34405
34411
|
const messages = normalizeKernelMessages(afterTurnMessages);
|
|
34406
34412
|
const msgCount = messages.length;
|
|
34407
34413
|
logger.info?.(
|
|
34408
|
-
`LibraVDB afterTurn sessionId=${
|
|
34414
|
+
`LibraVDB afterTurn sessionId=${sessionId} userId=${userId} messageCount=${msgCount} totalMessages=${args.messages.length} prePromptMessageCount=${args.prePromptMessageCount ?? "unknown"} heartbeat=${args.isHeartbeat ?? false}`
|
|
34409
34415
|
);
|
|
34410
34416
|
try {
|
|
34411
34417
|
const kernel = await getKernelOrNull("afterTurn");
|
|
@@ -34414,14 +34420,14 @@ function buildContextEngineFactory(runtime, cfg, logger = console) {
|
|
|
34414
34420
|
);
|
|
34415
34421
|
if (kernel) {
|
|
34416
34422
|
const result2 = await kernel.afterTurn({
|
|
34417
|
-
sessionId
|
|
34423
|
+
sessionId,
|
|
34418
34424
|
sessionKey: args.sessionKey,
|
|
34419
34425
|
userId,
|
|
34420
34426
|
messages,
|
|
34421
34427
|
isHeartbeat: args.isHeartbeat
|
|
34422
34428
|
});
|
|
34423
34429
|
await performAfterTurnPredictiveCompaction({
|
|
34424
|
-
sessionId
|
|
34430
|
+
sessionId,
|
|
34425
34431
|
tokenBudget: args.tokenBudget,
|
|
34426
34432
|
currentTokenCount
|
|
34427
34433
|
});
|
|
@@ -34429,21 +34435,21 @@ function buildContextEngineFactory(runtime, cfg, logger = console) {
|
|
|
34429
34435
|
}
|
|
34430
34436
|
const rpc = await runtime.getRpc();
|
|
34431
34437
|
const result = await rpc.call("after_turn_kernel", {
|
|
34432
|
-
sessionId
|
|
34438
|
+
sessionId,
|
|
34433
34439
|
sessionKey: args.sessionKey,
|
|
34434
34440
|
userId,
|
|
34435
34441
|
messages,
|
|
34436
34442
|
isHeartbeat: args.isHeartbeat
|
|
34437
34443
|
});
|
|
34438
34444
|
await performAfterTurnPredictiveCompaction({
|
|
34439
|
-
sessionId
|
|
34445
|
+
sessionId,
|
|
34440
34446
|
tokenBudget: args.tokenBudget,
|
|
34441
34447
|
currentTokenCount
|
|
34442
34448
|
});
|
|
34443
34449
|
return result;
|
|
34444
34450
|
} catch (error) {
|
|
34445
34451
|
logger.warn?.(
|
|
34446
|
-
`LibraVDB afterTurn failed sessionId=${
|
|
34452
|
+
`LibraVDB afterTurn failed sessionId=${sessionId}: ${error instanceof Error ? error.message : String(error)}`
|
|
34447
34453
|
);
|
|
34448
34454
|
throw error;
|
|
34449
34455
|
}
|
|
@@ -34517,6 +34523,7 @@ function isRecord(value) {
|
|
|
34517
34523
|
// src/markdown-ingest.ts
|
|
34518
34524
|
import fs2 from "node:fs";
|
|
34519
34525
|
import fsp2 from "node:fs/promises";
|
|
34526
|
+
import os2 from "node:os";
|
|
34520
34527
|
import path2 from "node:path";
|
|
34521
34528
|
|
|
34522
34529
|
// node_modules/.pnpm/@bufbuild+protobuf@1.7.2/node_modules/@bufbuild/protobuf/dist/proxy/index.js
|
|
@@ -38478,7 +38485,8 @@ function createMarkdownIngestionHandle(cfg, getRpc, logger = console, fsApi = cr
|
|
|
38478
38485
|
roots: genericRoots,
|
|
38479
38486
|
include: cfg.markdownIngestionInclude,
|
|
38480
38487
|
exclude: cfg.markdownIngestionExclude,
|
|
38481
|
-
debounceMs: cfg.markdownIngestionDebounceMs ?? DEFAULT_DEBOUNCE_MS2
|
|
38488
|
+
debounceMs: cfg.markdownIngestionDebounceMs ?? DEFAULT_DEBOUNCE_MS2,
|
|
38489
|
+
snapshotPath: resolveMarkdownSnapshotPath("generic", cfg.markdownIngestionSnapshotPath)
|
|
38482
38490
|
},
|
|
38483
38491
|
getRpc,
|
|
38484
38492
|
logger,
|
|
@@ -38487,7 +38495,7 @@ function createMarkdownIngestionHandle(cfg, getRpc, logger = console, fsApi = cr
|
|
|
38487
38495
|
);
|
|
38488
38496
|
}
|
|
38489
38497
|
const obsidianRoots = normalizeMarkdownRoots(cfg.markdownIngestionObsidianRoots);
|
|
38490
|
-
if (cfg.markdownIngestionObsidianEnabled
|
|
38498
|
+
if (cfg.markdownIngestionObsidianEnabled === true && obsidianRoots.length > 0) {
|
|
38491
38499
|
adapters.push(
|
|
38492
38500
|
new DirectoryMarkdownSourceAdapter(
|
|
38493
38501
|
"obsidian",
|
|
@@ -38495,7 +38503,8 @@ function createMarkdownIngestionHandle(cfg, getRpc, logger = console, fsApi = cr
|
|
|
38495
38503
|
roots: obsidianRoots,
|
|
38496
38504
|
include: cfg.markdownIngestionObsidianInclude,
|
|
38497
38505
|
exclude: cfg.markdownIngestionObsidianExclude,
|
|
38498
|
-
debounceMs: cfg.markdownIngestionObsidianDebounceMs ?? cfg.markdownIngestionDebounceMs ?? DEFAULT_DEBOUNCE_MS2
|
|
38506
|
+
debounceMs: cfg.markdownIngestionObsidianDebounceMs ?? cfg.markdownIngestionDebounceMs ?? DEFAULT_DEBOUNCE_MS2,
|
|
38507
|
+
snapshotPath: resolveMarkdownSnapshotPath("obsidian", cfg.markdownIngestionObsidianSnapshotPath)
|
|
38499
38508
|
},
|
|
38500
38509
|
getRpc,
|
|
38501
38510
|
logger,
|
|
@@ -38551,6 +38560,7 @@ var DirectoryMarkdownSourceAdapter = class {
|
|
|
38551
38560
|
fsApi;
|
|
38552
38561
|
getRpc;
|
|
38553
38562
|
logger;
|
|
38563
|
+
snapshotPath;
|
|
38554
38564
|
states = /* @__PURE__ */ new Map();
|
|
38555
38565
|
fileStates = /* @__PURE__ */ new Map();
|
|
38556
38566
|
activeScans = /* @__PURE__ */ new Set();
|
|
@@ -38559,6 +38569,8 @@ var DirectoryMarkdownSourceAdapter = class {
|
|
|
38559
38569
|
started = false;
|
|
38560
38570
|
ingestQueue = null;
|
|
38561
38571
|
stopping = false;
|
|
38572
|
+
snapshotLoaded = false;
|
|
38573
|
+
snapshotDirty = false;
|
|
38562
38574
|
constructor(kind, config, getRpc, logger, fsApi) {
|
|
38563
38575
|
this.kind = kind;
|
|
38564
38576
|
this.roots = config.roots;
|
|
@@ -38568,6 +38580,7 @@ var DirectoryMarkdownSourceAdapter = class {
|
|
|
38568
38580
|
this.fsApi = fsApi;
|
|
38569
38581
|
this.getRpc = getRpc;
|
|
38570
38582
|
this.logger = logger;
|
|
38583
|
+
this.snapshotPath = config.snapshotPath ?? resolveMarkdownSnapshotPath(kind);
|
|
38571
38584
|
this.tokenizerId = DEFAULT_TOKENIZER_ID;
|
|
38572
38585
|
this.coreDoc = true;
|
|
38573
38586
|
}
|
|
@@ -38575,6 +38588,7 @@ var DirectoryMarkdownSourceAdapter = class {
|
|
|
38575
38588
|
if (this.started) {
|
|
38576
38589
|
return;
|
|
38577
38590
|
}
|
|
38591
|
+
await this.loadSnapshot();
|
|
38578
38592
|
this.started = true;
|
|
38579
38593
|
this.stopping = false;
|
|
38580
38594
|
await this.refresh();
|
|
@@ -38602,8 +38616,10 @@ var DirectoryMarkdownSourceAdapter = class {
|
|
|
38602
38616
|
if (this.activeScans.size > 0) {
|
|
38603
38617
|
await Promise.allSettled([...this.activeScans]);
|
|
38604
38618
|
}
|
|
38619
|
+
await this.saveSnapshotIfDirty();
|
|
38605
38620
|
this.states.clear();
|
|
38606
38621
|
this.fileStates.clear();
|
|
38622
|
+
this.snapshotLoaded = false;
|
|
38607
38623
|
this.started = false;
|
|
38608
38624
|
}
|
|
38609
38625
|
getRootState(root) {
|
|
@@ -38619,7 +38635,7 @@ var DirectoryMarkdownSourceAdapter = class {
|
|
|
38619
38635
|
dirty: false,
|
|
38620
38636
|
timer: null
|
|
38621
38637
|
},
|
|
38622
|
-
knownFiles:
|
|
38638
|
+
knownFiles: this.snapshotFilesForRoot(resolved),
|
|
38623
38639
|
directoryWatchers: /* @__PURE__ */ new Map()
|
|
38624
38640
|
};
|
|
38625
38641
|
this.states.set(resolved, created);
|
|
@@ -38636,12 +38652,16 @@ var DirectoryMarkdownSourceAdapter = class {
|
|
|
38636
38652
|
}
|
|
38637
38653
|
rootState.scanState.scanning = true;
|
|
38638
38654
|
const scan = (async () => {
|
|
38655
|
+
const stats = createScanStats();
|
|
38656
|
+
const startedAt = Date.now();
|
|
38639
38657
|
try {
|
|
38640
38658
|
const currentFiles = /* @__PURE__ */ new Set();
|
|
38641
|
-
await this.walkDirectory(rootState, rootState.root, currentFiles);
|
|
38659
|
+
await this.walkDirectory(rootState, rootState.root, currentFiles, stats);
|
|
38642
38660
|
if (!this.stopping) {
|
|
38643
|
-
await this.pruneDeletedFiles(rootState, currentFiles);
|
|
38661
|
+
await this.pruneDeletedFiles(rootState, currentFiles, stats);
|
|
38644
38662
|
rootState.knownFiles = currentFiles;
|
|
38663
|
+
await this.saveSnapshotIfDirty();
|
|
38664
|
+
this.logScanStats(rootState.root, stats, Date.now() - startedAt);
|
|
38645
38665
|
}
|
|
38646
38666
|
} finally {
|
|
38647
38667
|
rootState.scanState.scanning = false;
|
|
@@ -38678,7 +38698,12 @@ var DirectoryMarkdownSourceAdapter = class {
|
|
|
38678
38698
|
});
|
|
38679
38699
|
}, this.debounceMs);
|
|
38680
38700
|
}
|
|
38681
|
-
async walkDirectory(rootState, dir, currentFiles) {
|
|
38701
|
+
async walkDirectory(rootState, dir, currentFiles, stats) {
|
|
38702
|
+
if (this.shouldPruneDirectory(rootState.root, dir)) {
|
|
38703
|
+
stats.directoriesPruned++;
|
|
38704
|
+
return;
|
|
38705
|
+
}
|
|
38706
|
+
stats.directoriesScanned++;
|
|
38682
38707
|
await this.ensureDirectoryWatcher(rootState, dir);
|
|
38683
38708
|
let entries;
|
|
38684
38709
|
try {
|
|
@@ -38696,25 +38721,42 @@ var DirectoryMarkdownSourceAdapter = class {
|
|
|
38696
38721
|
}
|
|
38697
38722
|
const child = path2.join(dir, entry.name);
|
|
38698
38723
|
if (entry.isDirectory()) {
|
|
38699
|
-
await this.walkDirectory(rootState, child, currentFiles);
|
|
38724
|
+
await this.walkDirectory(rootState, child, currentFiles, stats);
|
|
38700
38725
|
continue;
|
|
38701
38726
|
}
|
|
38702
38727
|
if (!entry.isFile() || !isMarkdownFile(entry.name)) {
|
|
38703
38728
|
continue;
|
|
38704
38729
|
}
|
|
38730
|
+
stats.markdownFilesSeen++;
|
|
38705
38731
|
if (!this.shouldIncludeFile(rootState.root, child)) {
|
|
38732
|
+
stats.filesSkipped++;
|
|
38706
38733
|
continue;
|
|
38707
38734
|
}
|
|
38735
|
+
stats.filesIncluded++;
|
|
38708
38736
|
currentFiles.add(child);
|
|
38709
38737
|
try {
|
|
38710
|
-
await this.syncMarkdownFile(rootState, child);
|
|
38738
|
+
const result = await this.syncMarkdownFile(rootState, child);
|
|
38739
|
+
recordSyncResult(stats, result);
|
|
38711
38740
|
} catch (error) {
|
|
38741
|
+
stats.syncErrors++;
|
|
38712
38742
|
if (!this.stopping) {
|
|
38713
38743
|
this.logger.warn?.(`[markdown-ingest] sync failed for ${child}: ${formatError(error)}`);
|
|
38714
38744
|
}
|
|
38715
38745
|
}
|
|
38716
38746
|
}
|
|
38717
38747
|
}
|
|
38748
|
+
shouldPruneDirectory(root, dir) {
|
|
38749
|
+
const relative = toPosixPath(path2.relative(root, dir));
|
|
38750
|
+
if (!relative || relative === "." || relative.startsWith("..")) {
|
|
38751
|
+
return false;
|
|
38752
|
+
}
|
|
38753
|
+
for (const pattern of this.excludePatterns) {
|
|
38754
|
+
if (matchesExcludedDirectory(relative, pattern)) {
|
|
38755
|
+
return true;
|
|
38756
|
+
}
|
|
38757
|
+
}
|
|
38758
|
+
return false;
|
|
38759
|
+
}
|
|
38718
38760
|
async ensureDirectoryWatcher(rootState, dir) {
|
|
38719
38761
|
if (rootState.directoryWatchers.has(dir)) {
|
|
38720
38762
|
return;
|
|
@@ -38755,7 +38797,7 @@ var DirectoryMarkdownSourceAdapter = class {
|
|
|
38755
38797
|
}
|
|
38756
38798
|
return true;
|
|
38757
38799
|
}
|
|
38758
|
-
async pruneDeletedFiles(rootState, currentFiles) {
|
|
38800
|
+
async pruneDeletedFiles(rootState, currentFiles, stats) {
|
|
38759
38801
|
const removed = [];
|
|
38760
38802
|
for (const previous of rootState.knownFiles) {
|
|
38761
38803
|
if (!currentFiles.has(previous)) {
|
|
@@ -38768,6 +38810,8 @@ var DirectoryMarkdownSourceAdapter = class {
|
|
|
38768
38810
|
for (const filePath of removed) {
|
|
38769
38811
|
await this.deleteSourceDocument(filePath);
|
|
38770
38812
|
this.fileStates.delete(filePath);
|
|
38813
|
+
this.snapshotDirty = true;
|
|
38814
|
+
stats.filesDeleted++;
|
|
38771
38815
|
}
|
|
38772
38816
|
}
|
|
38773
38817
|
async syncMarkdownFile(rootState, filePath) {
|
|
@@ -38777,21 +38821,23 @@ var DirectoryMarkdownSourceAdapter = class {
|
|
|
38777
38821
|
if (!stat) {
|
|
38778
38822
|
await this.deleteSourceDocument(sourceDoc);
|
|
38779
38823
|
this.fileStates.delete(sourceDoc);
|
|
38780
|
-
|
|
38824
|
+
this.snapshotDirty = true;
|
|
38825
|
+
return "deleted";
|
|
38781
38826
|
}
|
|
38782
38827
|
const cached = this.fileStates.get(sourceDoc);
|
|
38783
38828
|
if (cached && cached.size === stat.size && cached.mtimeMs === stat.mtimeMs) {
|
|
38784
|
-
return;
|
|
38829
|
+
return "unchanged";
|
|
38785
38830
|
}
|
|
38786
38831
|
const bytes = await this.safeReadFile(filePath);
|
|
38787
38832
|
if (!bytes) {
|
|
38788
38833
|
await this.deleteSourceDocument(sourceDoc);
|
|
38789
38834
|
this.fileStates.delete(sourceDoc);
|
|
38790
|
-
|
|
38835
|
+
this.snapshotDirty = true;
|
|
38836
|
+
return "deleted";
|
|
38791
38837
|
}
|
|
38792
38838
|
const fileHash = hashBytes(bytes);
|
|
38793
38839
|
if (cached && cached.fileHash === fileHash) {
|
|
38794
|
-
this.
|
|
38840
|
+
this.setFileState(sourceDoc, {
|
|
38795
38841
|
root: rootState.root,
|
|
38796
38842
|
sourceDoc,
|
|
38797
38843
|
relativePath,
|
|
@@ -38799,16 +38845,17 @@ var DirectoryMarkdownSourceAdapter = class {
|
|
|
38799
38845
|
size: stat.size,
|
|
38800
38846
|
mtimeMs: stat.mtimeMs
|
|
38801
38847
|
});
|
|
38802
|
-
return;
|
|
38848
|
+
return "unchanged";
|
|
38803
38849
|
}
|
|
38804
38850
|
const text = textDecoder2.decode(bytes);
|
|
38805
38851
|
if (this.kind === "obsidian" && this.includePatterns.length === 0 && !looksLikeObsidianNote(filePath, text)) {
|
|
38806
38852
|
await this.deleteSourceDocument(sourceDoc);
|
|
38807
38853
|
this.fileStates.delete(sourceDoc);
|
|
38808
|
-
|
|
38854
|
+
this.snapshotDirty = true;
|
|
38855
|
+
return "skipped";
|
|
38809
38856
|
}
|
|
38810
38857
|
await this.ingestMarkdownDocument(sourceDoc, text, rootState.root, relativePath, fileHash, stat.size, stat.mtimeMs);
|
|
38811
|
-
this.
|
|
38858
|
+
this.setFileState(sourceDoc, {
|
|
38812
38859
|
root: rootState.root,
|
|
38813
38860
|
sourceDoc,
|
|
38814
38861
|
relativePath,
|
|
@@ -38816,6 +38863,11 @@ var DirectoryMarkdownSourceAdapter = class {
|
|
|
38816
38863
|
size: stat.size,
|
|
38817
38864
|
mtimeMs: stat.mtimeMs
|
|
38818
38865
|
});
|
|
38866
|
+
return "ingested";
|
|
38867
|
+
}
|
|
38868
|
+
setFileState(sourceDoc, state) {
|
|
38869
|
+
this.fileStates.set(sourceDoc, state);
|
|
38870
|
+
this.snapshotDirty = true;
|
|
38819
38871
|
}
|
|
38820
38872
|
async ingestMarkdownDocument(sourceDoc, text, sourceRoot, sourcePath, fileHash, sourceSize, sourceMtimeMs) {
|
|
38821
38873
|
const queue = await this.getIngestQueue();
|
|
@@ -38863,7 +38915,96 @@ var DirectoryMarkdownSourceAdapter = class {
|
|
|
38863
38915
|
return null;
|
|
38864
38916
|
}
|
|
38865
38917
|
}
|
|
38918
|
+
snapshotFilesForRoot(root) {
|
|
38919
|
+
const files = /* @__PURE__ */ new Set();
|
|
38920
|
+
for (const state of this.fileStates.values()) {
|
|
38921
|
+
if (state.root === root) {
|
|
38922
|
+
files.add(state.sourceDoc);
|
|
38923
|
+
}
|
|
38924
|
+
}
|
|
38925
|
+
return files;
|
|
38926
|
+
}
|
|
38927
|
+
async loadSnapshot() {
|
|
38928
|
+
if (this.snapshotLoaded) {
|
|
38929
|
+
return;
|
|
38930
|
+
}
|
|
38931
|
+
this.snapshotLoaded = true;
|
|
38932
|
+
let raw;
|
|
38933
|
+
try {
|
|
38934
|
+
raw = await fsp2.readFile(this.snapshotPath, "utf8");
|
|
38935
|
+
} catch (error) {
|
|
38936
|
+
if (!formatError(error).includes("ENOENT")) {
|
|
38937
|
+
this.logger.warn?.(`[markdown-ingest] failed to read snapshot ${this.snapshotPath}: ${formatError(error)}`);
|
|
38938
|
+
}
|
|
38939
|
+
return;
|
|
38940
|
+
}
|
|
38941
|
+
try {
|
|
38942
|
+
const parsed = JSON.parse(raw);
|
|
38943
|
+
if (parsed.ingestVersion !== MARKDOWN_INGEST_VERSION || parsed.hashBackend !== HASH_BACKEND || !parsed.files) {
|
|
38944
|
+
return;
|
|
38945
|
+
}
|
|
38946
|
+
const configuredRoots = new Set(this.roots.map((root) => path2.resolve(root)));
|
|
38947
|
+
for (const [sourceDoc, state] of Object.entries(parsed.files)) {
|
|
38948
|
+
if (isValidSnapshotState(sourceDoc, state) && configuredRoots.has(path2.resolve(state.root))) {
|
|
38949
|
+
this.fileStates.set(sourceDoc, state);
|
|
38950
|
+
}
|
|
38951
|
+
}
|
|
38952
|
+
this.logger.info?.(`[markdown-ingest] loaded ${this.fileStates.size} ${this.kind} file snapshots from ${this.snapshotPath}`);
|
|
38953
|
+
} catch (error) {
|
|
38954
|
+
this.logger.warn?.(`[markdown-ingest] failed to parse snapshot ${this.snapshotPath}: ${formatError(error)}`);
|
|
38955
|
+
}
|
|
38956
|
+
}
|
|
38957
|
+
async saveSnapshotIfDirty() {
|
|
38958
|
+
if (!this.snapshotDirty) {
|
|
38959
|
+
return;
|
|
38960
|
+
}
|
|
38961
|
+
const payload = {
|
|
38962
|
+
version: 1,
|
|
38963
|
+
ingestVersion: MARKDOWN_INGEST_VERSION,
|
|
38964
|
+
hashBackend: HASH_BACKEND,
|
|
38965
|
+
files: Object.fromEntries([...this.fileStates.entries()].sort(([left], [right]) => left.localeCompare(right)))
|
|
38966
|
+
};
|
|
38967
|
+
try {
|
|
38968
|
+
await fsp2.mkdir(path2.dirname(this.snapshotPath), { recursive: true });
|
|
38969
|
+
const tmp = `${this.snapshotPath}.${process.pid}.${Math.random().toString(36).slice(2, 8)}.tmp`;
|
|
38970
|
+
await fsp2.writeFile(tmp, `${JSON.stringify(payload, null, 2)}
|
|
38971
|
+
`);
|
|
38972
|
+
await fsp2.rename(tmp, this.snapshotPath);
|
|
38973
|
+
this.snapshotDirty = false;
|
|
38974
|
+
} catch (error) {
|
|
38975
|
+
this.logger.warn?.(`[markdown-ingest] failed to write snapshot ${this.snapshotPath}: ${formatError(error)}`);
|
|
38976
|
+
}
|
|
38977
|
+
}
|
|
38978
|
+
logScanStats(root, stats, durationMs) {
|
|
38979
|
+
this.logger.info?.(
|
|
38980
|
+
`[markdown-ingest] ${this.kind} scan complete root=${root} dirs=${stats.directoriesScanned} prunedDirs=${stats.directoriesPruned} markdown=${stats.markdownFilesSeen} included=${stats.filesIncluded} skipped=${stats.filesSkipped} unchanged=${stats.filesUnchanged} ingested=${stats.filesIngested} deleted=${stats.filesDeleted} errors=${stats.syncErrors} durationMs=${durationMs}`
|
|
38981
|
+
);
|
|
38982
|
+
}
|
|
38866
38983
|
};
|
|
38984
|
+
function createScanStats() {
|
|
38985
|
+
return {
|
|
38986
|
+
directoriesScanned: 0,
|
|
38987
|
+
directoriesPruned: 0,
|
|
38988
|
+
markdownFilesSeen: 0,
|
|
38989
|
+
filesIncluded: 0,
|
|
38990
|
+
filesSkipped: 0,
|
|
38991
|
+
filesUnchanged: 0,
|
|
38992
|
+
filesIngested: 0,
|
|
38993
|
+
filesDeleted: 0,
|
|
38994
|
+
syncErrors: 0
|
|
38995
|
+
};
|
|
38996
|
+
}
|
|
38997
|
+
function recordSyncResult(stats, result) {
|
|
38998
|
+
if (result === "ingested") {
|
|
38999
|
+
stats.filesIngested++;
|
|
39000
|
+
} else if (result === "unchanged") {
|
|
39001
|
+
stats.filesUnchanged++;
|
|
39002
|
+
} else if (result === "deleted") {
|
|
39003
|
+
stats.filesDeleted++;
|
|
39004
|
+
} else {
|
|
39005
|
+
stats.filesSkipped++;
|
|
39006
|
+
}
|
|
39007
|
+
}
|
|
38867
39008
|
function toPosixPath(value) {
|
|
38868
39009
|
return value.split(path2.sep).join("/");
|
|
38869
39010
|
}
|
|
@@ -38882,11 +39023,16 @@ function normalizeMarkdownRoots(roots) {
|
|
|
38882
39023
|
}
|
|
38883
39024
|
return [...resolved];
|
|
38884
39025
|
}
|
|
38885
|
-
function
|
|
38886
|
-
|
|
38887
|
-
|
|
39026
|
+
function resolveMarkdownSnapshotPath(kind, configuredPath) {
|
|
39027
|
+
const trimmed = configuredPath?.trim();
|
|
39028
|
+
if (trimmed) {
|
|
39029
|
+
return path2.resolve(trimmed);
|
|
38888
39030
|
}
|
|
38889
|
-
|
|
39031
|
+
const stateDir = process.env.OPENCLAW_STATE_DIR?.trim() || path2.join(os2.homedir(), ".openclaw");
|
|
39032
|
+
return path2.join(stateDir, `libravdb-markdown-ingest-${kind}.json`);
|
|
39033
|
+
}
|
|
39034
|
+
function isMarkdownIngestionEnabled(cfg, roots) {
|
|
39035
|
+
return cfg.markdownIngestionEnabled === true && roots.length > 0;
|
|
38890
39036
|
}
|
|
38891
39037
|
function createRealFsApi2() {
|
|
38892
39038
|
return {
|
|
@@ -38907,6 +39053,17 @@ function matchesGlob(value, pattern) {
|
|
|
38907
39053
|
const escaped = pattern.split("*").map((part) => part.replace(/[.+?^${}()|[\]\\]/g, "\\$&")).join(".*");
|
|
38908
39054
|
return new RegExp(`^${escaped}$`).test(value);
|
|
38909
39055
|
}
|
|
39056
|
+
function matchesExcludedDirectory(relativeDir, pattern) {
|
|
39057
|
+
const normalized = relativeDir.replace(/\/+$/, "");
|
|
39058
|
+
return matchesGlob(normalized, pattern) || matchesGlob(`${normalized}/`, pattern) || matchesGlob(`${normalized}/.probe`, pattern);
|
|
39059
|
+
}
|
|
39060
|
+
function isValidSnapshotState(sourceDoc, value) {
|
|
39061
|
+
if (!value || typeof value !== "object") {
|
|
39062
|
+
return false;
|
|
39063
|
+
}
|
|
39064
|
+
const state = value;
|
|
39065
|
+
return state.sourceDoc === sourceDoc && typeof state.root === "string" && typeof state.relativePath === "string" && typeof state.fileHash === "string" && typeof state.size === "number" && Number.isFinite(state.size) && typeof state.mtimeMs === "number" && Number.isFinite(state.mtimeMs);
|
|
39066
|
+
}
|
|
38910
39067
|
function looksLikeObsidianNote(filePath, text) {
|
|
38911
39068
|
const frontmatterStart = parseFrontmatterStart(text);
|
|
38912
39069
|
if (frontmatterStart == null) {
|
|
@@ -39433,7 +39590,7 @@ var GrpcKernelClient = class {
|
|
|
39433
39590
|
// src/sidecar.ts
|
|
39434
39591
|
import fs3 from "node:fs";
|
|
39435
39592
|
import net from "node:net";
|
|
39436
|
-
import
|
|
39593
|
+
import os3 from "node:os";
|
|
39437
39594
|
import path4 from "node:path";
|
|
39438
39595
|
var STARTUP_CONNECT_MAX_RETRIES = 5;
|
|
39439
39596
|
var STARTUP_CONNECT_BASE_DELAY_MS = 100;
|
|
@@ -39704,7 +39861,7 @@ function resolveConfiguredEndpoint(cfg) {
|
|
|
39704
39861
|
function daemonProvisioningHint() {
|
|
39705
39862
|
return "If you installed the npm package, install and start libravdbd separately; the package does not provision the daemon binary, ONNX Runtime, or model assets.";
|
|
39706
39863
|
}
|
|
39707
|
-
function defaultEndpoint(platform = process.platform, homeDir =
|
|
39864
|
+
function defaultEndpoint(platform = process.platform, homeDir = os3.homedir(), pathExists = fs3.existsSync) {
|
|
39708
39865
|
const envEndpoint = normalizeConfiguredEndpoint(process.env.LIBRAVDB_RPC_ENDPOINT);
|
|
39709
39866
|
if (envEndpoint) {
|
|
39710
39867
|
return envEndpoint;
|
package/dist/markdown-ingest.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import fs from "node:fs";
|
|
2
2
|
import fsp from "node:fs/promises";
|
|
3
|
+
import os from "node:os";
|
|
3
4
|
import path from "node:path";
|
|
4
5
|
import { hashBytes } from "./markdown-hash.js";
|
|
5
6
|
import { formatError } from "./format-error.js";
|
|
@@ -17,15 +18,17 @@ export function createMarkdownIngestionHandle(cfg, getRpc, logger = console, fsA
|
|
|
17
18
|
include: cfg.markdownIngestionInclude,
|
|
18
19
|
exclude: cfg.markdownIngestionExclude,
|
|
19
20
|
debounceMs: cfg.markdownIngestionDebounceMs ?? DEFAULT_DEBOUNCE_MS,
|
|
21
|
+
snapshotPath: resolveMarkdownSnapshotPath("generic", cfg.markdownIngestionSnapshotPath),
|
|
20
22
|
}, getRpc, logger, fsApi));
|
|
21
23
|
}
|
|
22
24
|
const obsidianRoots = normalizeMarkdownRoots(cfg.markdownIngestionObsidianRoots);
|
|
23
|
-
if (cfg.markdownIngestionObsidianEnabled
|
|
25
|
+
if (cfg.markdownIngestionObsidianEnabled === true && obsidianRoots.length > 0) {
|
|
24
26
|
adapters.push(new DirectoryMarkdownSourceAdapter("obsidian", {
|
|
25
27
|
roots: obsidianRoots,
|
|
26
28
|
include: cfg.markdownIngestionObsidianInclude,
|
|
27
29
|
exclude: cfg.markdownIngestionObsidianExclude,
|
|
28
30
|
debounceMs: cfg.markdownIngestionObsidianDebounceMs ?? cfg.markdownIngestionDebounceMs ?? DEFAULT_DEBOUNCE_MS,
|
|
31
|
+
snapshotPath: resolveMarkdownSnapshotPath("obsidian", cfg.markdownIngestionObsidianSnapshotPath),
|
|
29
32
|
}, getRpc, logger, fsApi));
|
|
30
33
|
}
|
|
31
34
|
if (adapters.length === 0) {
|
|
@@ -73,6 +76,7 @@ class DirectoryMarkdownSourceAdapter {
|
|
|
73
76
|
fsApi;
|
|
74
77
|
getRpc;
|
|
75
78
|
logger;
|
|
79
|
+
snapshotPath;
|
|
76
80
|
states = new Map();
|
|
77
81
|
fileStates = new Map();
|
|
78
82
|
activeScans = new Set();
|
|
@@ -81,6 +85,8 @@ class DirectoryMarkdownSourceAdapter {
|
|
|
81
85
|
started = false;
|
|
82
86
|
ingestQueue = null;
|
|
83
87
|
stopping = false;
|
|
88
|
+
snapshotLoaded = false;
|
|
89
|
+
snapshotDirty = false;
|
|
84
90
|
constructor(kind, config, getRpc, logger, fsApi) {
|
|
85
91
|
this.kind = kind;
|
|
86
92
|
this.roots = config.roots;
|
|
@@ -90,6 +96,7 @@ class DirectoryMarkdownSourceAdapter {
|
|
|
90
96
|
this.fsApi = fsApi;
|
|
91
97
|
this.getRpc = getRpc;
|
|
92
98
|
this.logger = logger;
|
|
99
|
+
this.snapshotPath = config.snapshotPath ?? resolveMarkdownSnapshotPath(kind);
|
|
93
100
|
this.tokenizerId = DEFAULT_TOKENIZER_ID;
|
|
94
101
|
this.coreDoc = true;
|
|
95
102
|
}
|
|
@@ -97,6 +104,7 @@ class DirectoryMarkdownSourceAdapter {
|
|
|
97
104
|
if (this.started) {
|
|
98
105
|
return;
|
|
99
106
|
}
|
|
107
|
+
await this.loadSnapshot();
|
|
100
108
|
this.started = true;
|
|
101
109
|
this.stopping = false;
|
|
102
110
|
await this.refresh();
|
|
@@ -124,8 +132,10 @@ class DirectoryMarkdownSourceAdapter {
|
|
|
124
132
|
if (this.activeScans.size > 0) {
|
|
125
133
|
await Promise.allSettled([...this.activeScans]);
|
|
126
134
|
}
|
|
135
|
+
await this.saveSnapshotIfDirty();
|
|
127
136
|
this.states.clear();
|
|
128
137
|
this.fileStates.clear();
|
|
138
|
+
this.snapshotLoaded = false;
|
|
129
139
|
this.started = false;
|
|
130
140
|
}
|
|
131
141
|
getRootState(root) {
|
|
@@ -141,7 +151,7 @@ class DirectoryMarkdownSourceAdapter {
|
|
|
141
151
|
dirty: false,
|
|
142
152
|
timer: null,
|
|
143
153
|
},
|
|
144
|
-
knownFiles:
|
|
154
|
+
knownFiles: this.snapshotFilesForRoot(resolved),
|
|
145
155
|
directoryWatchers: new Map(),
|
|
146
156
|
};
|
|
147
157
|
this.states.set(resolved, created);
|
|
@@ -158,12 +168,16 @@ class DirectoryMarkdownSourceAdapter {
|
|
|
158
168
|
}
|
|
159
169
|
rootState.scanState.scanning = true;
|
|
160
170
|
const scan = (async () => {
|
|
171
|
+
const stats = createScanStats();
|
|
172
|
+
const startedAt = Date.now();
|
|
161
173
|
try {
|
|
162
174
|
const currentFiles = new Set();
|
|
163
|
-
await this.walkDirectory(rootState, rootState.root, currentFiles);
|
|
175
|
+
await this.walkDirectory(rootState, rootState.root, currentFiles, stats);
|
|
164
176
|
if (!this.stopping) {
|
|
165
|
-
await this.pruneDeletedFiles(rootState, currentFiles);
|
|
177
|
+
await this.pruneDeletedFiles(rootState, currentFiles, stats);
|
|
166
178
|
rootState.knownFiles = currentFiles;
|
|
179
|
+
await this.saveSnapshotIfDirty();
|
|
180
|
+
this.logScanStats(rootState.root, stats, Date.now() - startedAt);
|
|
167
181
|
}
|
|
168
182
|
}
|
|
169
183
|
finally {
|
|
@@ -202,7 +216,12 @@ class DirectoryMarkdownSourceAdapter {
|
|
|
202
216
|
});
|
|
203
217
|
}, this.debounceMs);
|
|
204
218
|
}
|
|
205
|
-
async walkDirectory(rootState, dir, currentFiles) {
|
|
219
|
+
async walkDirectory(rootState, dir, currentFiles, stats) {
|
|
220
|
+
if (this.shouldPruneDirectory(rootState.root, dir)) {
|
|
221
|
+
stats.directoriesPruned++;
|
|
222
|
+
return;
|
|
223
|
+
}
|
|
224
|
+
stats.directoriesScanned++;
|
|
206
225
|
await this.ensureDirectoryWatcher(rootState, dir);
|
|
207
226
|
let entries;
|
|
208
227
|
try {
|
|
@@ -221,26 +240,43 @@ class DirectoryMarkdownSourceAdapter {
|
|
|
221
240
|
}
|
|
222
241
|
const child = path.join(dir, entry.name);
|
|
223
242
|
if (entry.isDirectory()) {
|
|
224
|
-
await this.walkDirectory(rootState, child, currentFiles);
|
|
243
|
+
await this.walkDirectory(rootState, child, currentFiles, stats);
|
|
225
244
|
continue;
|
|
226
245
|
}
|
|
227
246
|
if (!entry.isFile() || !isMarkdownFile(entry.name)) {
|
|
228
247
|
continue;
|
|
229
248
|
}
|
|
249
|
+
stats.markdownFilesSeen++;
|
|
230
250
|
if (!this.shouldIncludeFile(rootState.root, child)) {
|
|
251
|
+
stats.filesSkipped++;
|
|
231
252
|
continue;
|
|
232
253
|
}
|
|
254
|
+
stats.filesIncluded++;
|
|
233
255
|
currentFiles.add(child);
|
|
234
256
|
try {
|
|
235
|
-
await this.syncMarkdownFile(rootState, child);
|
|
257
|
+
const result = await this.syncMarkdownFile(rootState, child);
|
|
258
|
+
recordSyncResult(stats, result);
|
|
236
259
|
}
|
|
237
260
|
catch (error) {
|
|
261
|
+
stats.syncErrors++;
|
|
238
262
|
if (!this.stopping) {
|
|
239
263
|
this.logger.warn?.(`[markdown-ingest] sync failed for ${child}: ${formatError(error)}`);
|
|
240
264
|
}
|
|
241
265
|
}
|
|
242
266
|
}
|
|
243
267
|
}
|
|
268
|
+
shouldPruneDirectory(root, dir) {
|
|
269
|
+
const relative = toPosixPath(path.relative(root, dir));
|
|
270
|
+
if (!relative || relative === "." || relative.startsWith("..")) {
|
|
271
|
+
return false;
|
|
272
|
+
}
|
|
273
|
+
for (const pattern of this.excludePatterns) {
|
|
274
|
+
if (matchesExcludedDirectory(relative, pattern)) {
|
|
275
|
+
return true;
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
return false;
|
|
279
|
+
}
|
|
244
280
|
async ensureDirectoryWatcher(rootState, dir) {
|
|
245
281
|
if (rootState.directoryWatchers.has(dir)) {
|
|
246
282
|
return;
|
|
@@ -282,7 +318,7 @@ class DirectoryMarkdownSourceAdapter {
|
|
|
282
318
|
}
|
|
283
319
|
return true;
|
|
284
320
|
}
|
|
285
|
-
async pruneDeletedFiles(rootState, currentFiles) {
|
|
321
|
+
async pruneDeletedFiles(rootState, currentFiles, stats) {
|
|
286
322
|
const removed = [];
|
|
287
323
|
for (const previous of rootState.knownFiles) {
|
|
288
324
|
if (!currentFiles.has(previous)) {
|
|
@@ -295,6 +331,8 @@ class DirectoryMarkdownSourceAdapter {
|
|
|
295
331
|
for (const filePath of removed) {
|
|
296
332
|
await this.deleteSourceDocument(filePath);
|
|
297
333
|
this.fileStates.delete(filePath);
|
|
334
|
+
this.snapshotDirty = true;
|
|
335
|
+
stats.filesDeleted++;
|
|
298
336
|
}
|
|
299
337
|
}
|
|
300
338
|
async syncMarkdownFile(rootState, filePath) {
|
|
@@ -304,21 +342,23 @@ class DirectoryMarkdownSourceAdapter {
|
|
|
304
342
|
if (!stat) {
|
|
305
343
|
await this.deleteSourceDocument(sourceDoc);
|
|
306
344
|
this.fileStates.delete(sourceDoc);
|
|
307
|
-
|
|
345
|
+
this.snapshotDirty = true;
|
|
346
|
+
return "deleted";
|
|
308
347
|
}
|
|
309
348
|
const cached = this.fileStates.get(sourceDoc);
|
|
310
349
|
if (cached && cached.size === stat.size && cached.mtimeMs === stat.mtimeMs) {
|
|
311
|
-
return;
|
|
350
|
+
return "unchanged";
|
|
312
351
|
}
|
|
313
352
|
const bytes = await this.safeReadFile(filePath);
|
|
314
353
|
if (!bytes) {
|
|
315
354
|
await this.deleteSourceDocument(sourceDoc);
|
|
316
355
|
this.fileStates.delete(sourceDoc);
|
|
317
|
-
|
|
356
|
+
this.snapshotDirty = true;
|
|
357
|
+
return "deleted";
|
|
318
358
|
}
|
|
319
359
|
const fileHash = hashBytes(bytes);
|
|
320
360
|
if (cached && cached.fileHash === fileHash) {
|
|
321
|
-
this.
|
|
361
|
+
this.setFileState(sourceDoc, {
|
|
322
362
|
root: rootState.root,
|
|
323
363
|
sourceDoc,
|
|
324
364
|
relativePath,
|
|
@@ -326,16 +366,17 @@ class DirectoryMarkdownSourceAdapter {
|
|
|
326
366
|
size: stat.size,
|
|
327
367
|
mtimeMs: stat.mtimeMs,
|
|
328
368
|
});
|
|
329
|
-
return;
|
|
369
|
+
return "unchanged";
|
|
330
370
|
}
|
|
331
371
|
const text = textDecoder.decode(bytes);
|
|
332
372
|
if (this.kind === "obsidian" && this.includePatterns.length === 0 && !looksLikeObsidianNote(filePath, text)) {
|
|
333
373
|
await this.deleteSourceDocument(sourceDoc);
|
|
334
374
|
this.fileStates.delete(sourceDoc);
|
|
335
|
-
|
|
375
|
+
this.snapshotDirty = true;
|
|
376
|
+
return "skipped";
|
|
336
377
|
}
|
|
337
378
|
await this.ingestMarkdownDocument(sourceDoc, text, rootState.root, relativePath, fileHash, stat.size, stat.mtimeMs);
|
|
338
|
-
this.
|
|
379
|
+
this.setFileState(sourceDoc, {
|
|
339
380
|
root: rootState.root,
|
|
340
381
|
sourceDoc,
|
|
341
382
|
relativePath,
|
|
@@ -343,6 +384,11 @@ class DirectoryMarkdownSourceAdapter {
|
|
|
343
384
|
size: stat.size,
|
|
344
385
|
mtimeMs: stat.mtimeMs,
|
|
345
386
|
});
|
|
387
|
+
return "ingested";
|
|
388
|
+
}
|
|
389
|
+
setFileState(sourceDoc, state) {
|
|
390
|
+
this.fileStates.set(sourceDoc, state);
|
|
391
|
+
this.snapshotDirty = true;
|
|
346
392
|
}
|
|
347
393
|
async ingestMarkdownDocument(sourceDoc, text, sourceRoot, sourcePath, fileHash, sourceSize, sourceMtimeMs) {
|
|
348
394
|
const queue = await this.getIngestQueue();
|
|
@@ -388,6 +434,98 @@ class DirectoryMarkdownSourceAdapter {
|
|
|
388
434
|
return null;
|
|
389
435
|
}
|
|
390
436
|
}
|
|
437
|
+
snapshotFilesForRoot(root) {
|
|
438
|
+
const files = new Set();
|
|
439
|
+
for (const state of this.fileStates.values()) {
|
|
440
|
+
if (state.root === root) {
|
|
441
|
+
files.add(state.sourceDoc);
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
return files;
|
|
445
|
+
}
|
|
446
|
+
async loadSnapshot() {
|
|
447
|
+
if (this.snapshotLoaded) {
|
|
448
|
+
return;
|
|
449
|
+
}
|
|
450
|
+
this.snapshotLoaded = true;
|
|
451
|
+
let raw;
|
|
452
|
+
try {
|
|
453
|
+
raw = await fsp.readFile(this.snapshotPath, "utf8");
|
|
454
|
+
}
|
|
455
|
+
catch (error) {
|
|
456
|
+
if (!formatError(error).includes("ENOENT")) {
|
|
457
|
+
this.logger.warn?.(`[markdown-ingest] failed to read snapshot ${this.snapshotPath}: ${formatError(error)}`);
|
|
458
|
+
}
|
|
459
|
+
return;
|
|
460
|
+
}
|
|
461
|
+
try {
|
|
462
|
+
const parsed = JSON.parse(raw);
|
|
463
|
+
if (parsed.ingestVersion !== MARKDOWN_INGEST_VERSION || parsed.hashBackend !== HASH_BACKEND || !parsed.files) {
|
|
464
|
+
return;
|
|
465
|
+
}
|
|
466
|
+
const configuredRoots = new Set(this.roots.map((root) => path.resolve(root)));
|
|
467
|
+
for (const [sourceDoc, state] of Object.entries(parsed.files)) {
|
|
468
|
+
if (isValidSnapshotState(sourceDoc, state) && configuredRoots.has(path.resolve(state.root))) {
|
|
469
|
+
this.fileStates.set(sourceDoc, state);
|
|
470
|
+
}
|
|
471
|
+
}
|
|
472
|
+
this.logger.info?.(`[markdown-ingest] loaded ${this.fileStates.size} ${this.kind} file snapshots from ${this.snapshotPath}`);
|
|
473
|
+
}
|
|
474
|
+
catch (error) {
|
|
475
|
+
this.logger.warn?.(`[markdown-ingest] failed to parse snapshot ${this.snapshotPath}: ${formatError(error)}`);
|
|
476
|
+
}
|
|
477
|
+
}
|
|
478
|
+
async saveSnapshotIfDirty() {
|
|
479
|
+
if (!this.snapshotDirty) {
|
|
480
|
+
return;
|
|
481
|
+
}
|
|
482
|
+
const payload = {
|
|
483
|
+
version: 1,
|
|
484
|
+
ingestVersion: MARKDOWN_INGEST_VERSION,
|
|
485
|
+
hashBackend: HASH_BACKEND,
|
|
486
|
+
files: Object.fromEntries([...this.fileStates.entries()].sort(([left], [right]) => left.localeCompare(right))),
|
|
487
|
+
};
|
|
488
|
+
try {
|
|
489
|
+
await fsp.mkdir(path.dirname(this.snapshotPath), { recursive: true });
|
|
490
|
+
const tmp = `${this.snapshotPath}.${process.pid}.${Math.random().toString(36).slice(2, 8)}.tmp`;
|
|
491
|
+
await fsp.writeFile(tmp, `${JSON.stringify(payload, null, 2)}\n`);
|
|
492
|
+
await fsp.rename(tmp, this.snapshotPath);
|
|
493
|
+
this.snapshotDirty = false;
|
|
494
|
+
}
|
|
495
|
+
catch (error) {
|
|
496
|
+
this.logger.warn?.(`[markdown-ingest] failed to write snapshot ${this.snapshotPath}: ${formatError(error)}`);
|
|
497
|
+
}
|
|
498
|
+
}
|
|
499
|
+
logScanStats(root, stats, durationMs) {
|
|
500
|
+
this.logger.info?.(`[markdown-ingest] ${this.kind} scan complete root=${root} dirs=${stats.directoriesScanned} prunedDirs=${stats.directoriesPruned} markdown=${stats.markdownFilesSeen} included=${stats.filesIncluded} skipped=${stats.filesSkipped} unchanged=${stats.filesUnchanged} ingested=${stats.filesIngested} deleted=${stats.filesDeleted} errors=${stats.syncErrors} durationMs=${durationMs}`);
|
|
501
|
+
}
|
|
502
|
+
}
|
|
503
|
+
function createScanStats() {
|
|
504
|
+
return {
|
|
505
|
+
directoriesScanned: 0,
|
|
506
|
+
directoriesPruned: 0,
|
|
507
|
+
markdownFilesSeen: 0,
|
|
508
|
+
filesIncluded: 0,
|
|
509
|
+
filesSkipped: 0,
|
|
510
|
+
filesUnchanged: 0,
|
|
511
|
+
filesIngested: 0,
|
|
512
|
+
filesDeleted: 0,
|
|
513
|
+
syncErrors: 0,
|
|
514
|
+
};
|
|
515
|
+
}
|
|
516
|
+
function recordSyncResult(stats, result) {
|
|
517
|
+
if (result === "ingested") {
|
|
518
|
+
stats.filesIngested++;
|
|
519
|
+
}
|
|
520
|
+
else if (result === "unchanged") {
|
|
521
|
+
stats.filesUnchanged++;
|
|
522
|
+
}
|
|
523
|
+
else if (result === "deleted") {
|
|
524
|
+
stats.filesDeleted++;
|
|
525
|
+
}
|
|
526
|
+
else {
|
|
527
|
+
stats.filesSkipped++;
|
|
528
|
+
}
|
|
391
529
|
}
|
|
392
530
|
function toPosixPath(value) {
|
|
393
531
|
return value.split(path.sep).join("/");
|
|
@@ -407,11 +545,16 @@ function normalizeMarkdownRoots(roots) {
|
|
|
407
545
|
}
|
|
408
546
|
return [...resolved];
|
|
409
547
|
}
|
|
410
|
-
function
|
|
411
|
-
|
|
412
|
-
|
|
548
|
+
function resolveMarkdownSnapshotPath(kind, configuredPath) {
|
|
549
|
+
const trimmed = configuredPath?.trim();
|
|
550
|
+
if (trimmed) {
|
|
551
|
+
return path.resolve(trimmed);
|
|
413
552
|
}
|
|
414
|
-
|
|
553
|
+
const stateDir = process.env.OPENCLAW_STATE_DIR?.trim() || path.join(os.homedir(), ".openclaw");
|
|
554
|
+
return path.join(stateDir, `libravdb-markdown-ingest-${kind}.json`);
|
|
555
|
+
}
|
|
556
|
+
function isMarkdownIngestionEnabled(cfg, roots) {
|
|
557
|
+
return cfg.markdownIngestionEnabled === true && roots.length > 0;
|
|
415
558
|
}
|
|
416
559
|
function createRealFsApi() {
|
|
417
560
|
return {
|
|
@@ -435,6 +578,24 @@ function matchesGlob(value, pattern) {
|
|
|
435
578
|
.join(".*");
|
|
436
579
|
return new RegExp(`^${escaped}$`).test(value);
|
|
437
580
|
}
|
|
581
|
+
function matchesExcludedDirectory(relativeDir, pattern) {
|
|
582
|
+
const normalized = relativeDir.replace(/\/+$/, "");
|
|
583
|
+
return matchesGlob(normalized, pattern) || matchesGlob(`${normalized}/`, pattern) || matchesGlob(`${normalized}/.probe`, pattern);
|
|
584
|
+
}
|
|
585
|
+
function isValidSnapshotState(sourceDoc, value) {
|
|
586
|
+
if (!value || typeof value !== "object") {
|
|
587
|
+
return false;
|
|
588
|
+
}
|
|
589
|
+
const state = value;
|
|
590
|
+
return (state.sourceDoc === sourceDoc &&
|
|
591
|
+
typeof state.root === "string" &&
|
|
592
|
+
typeof state.relativePath === "string" &&
|
|
593
|
+
typeof state.fileHash === "string" &&
|
|
594
|
+
typeof state.size === "number" &&
|
|
595
|
+
Number.isFinite(state.size) &&
|
|
596
|
+
typeof state.mtimeMs === "number" &&
|
|
597
|
+
Number.isFinite(state.mtimeMs));
|
|
598
|
+
}
|
|
438
599
|
function looksLikeObsidianNote(filePath, text) {
|
|
439
600
|
const frontmatterStart = parseFrontmatterStart(text);
|
|
440
601
|
if (frontmatterStart == null) {
|
package/dist/types.d.ts
CHANGED
|
@@ -48,6 +48,8 @@ export interface PluginConfig {
|
|
|
48
48
|
markdownIngestionInclude?: string[];
|
|
49
49
|
markdownIngestionExclude?: string[];
|
|
50
50
|
markdownIngestionDebounceMs?: number;
|
|
51
|
+
markdownIngestionSnapshotPath?: string;
|
|
52
|
+
markdownIngestionObsidianSnapshotPath?: string;
|
|
51
53
|
dreamPromotionEnabled?: boolean;
|
|
52
54
|
dreamPromotionDiaryPath?: string;
|
|
53
55
|
dreamPromotionUserId?: string;
|
package/docs/configuration.md
CHANGED
|
@@ -98,6 +98,10 @@ The plugin exposes `ingestionGateThreshold` for host-side gating decisions:
|
|
|
98
98
|
| `markdownIngestionObsidianExclude` | string[] | — | Obsidian glob exclude patterns |
|
|
99
99
|
| `markdownIngestionObsidianDebounceMs` | number | `150` | Obsidian debounce window |
|
|
100
100
|
|
|
101
|
+
Configured markdown roots are ignored unless the matching enable flag is set to
|
|
102
|
+
`true`. Set `markdownIngestionEnabled: true` for generic roots and
|
|
103
|
+
`markdownIngestionObsidianEnabled: true` for Obsidian vault roots.
|
|
104
|
+
|
|
101
105
|
## Dream promotion
|
|
102
106
|
|
|
103
107
|
| Key | Type | Default | Notes |
|
package/openclaw.plugin.json
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"id": "libravdb-memory",
|
|
3
3
|
"name": "LibraVDB Memory",
|
|
4
4
|
"description": "Persistent vector memory with three-tier hybrid scoring",
|
|
5
|
-
"version": "1.4.
|
|
5
|
+
"version": "1.4.68",
|
|
6
6
|
"kind": [
|
|
7
7
|
"memory",
|
|
8
8
|
"context-engine"
|
|
@@ -206,6 +206,12 @@
|
|
|
206
206
|
"type": "number",
|
|
207
207
|
"default": 150
|
|
208
208
|
},
|
|
209
|
+
"markdownIngestionSnapshotPath": {
|
|
210
|
+
"type": "string"
|
|
211
|
+
},
|
|
212
|
+
"markdownIngestionObsidianSnapshotPath": {
|
|
213
|
+
"type": "string"
|
|
214
|
+
},
|
|
209
215
|
"dreamPromotionEnabled": {
|
|
210
216
|
"type": "boolean",
|
|
211
217
|
"default": false
|