@yamo/memory-mesh 3.2.4 → 3.2.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -85,6 +85,11 @@ export declare class LanceDBClient {
85
85
  * @throws {QueryError} If stats query fails
86
86
  */
87
87
  getStats(): Promise<any>;
88
+ /**
89
+ * Compact old data files and prune versions older than 7 days.
90
+ * Best-effort — never throws.
91
+ */
92
+ optimize(): Promise<void>;
88
93
  /**
89
94
  * Sanitize an ID to prevent SQL injection
90
95
  * Removes any characters that aren't alphanumeric, underscore, or hyphen
@@ -201,7 +201,7 @@ export class LanceDBClient {
201
201
  await this.connect();
202
202
  }
203
203
  this._validateVector(vector);
204
- const { limit = 10, nprobes = 20, filter = null } = options;
204
+ const { limit = 10, nprobes = 20, filter = null, refineFactor, timeoutMs } = options;
205
205
  return this._retryOperation(async () => {
206
206
  if (!this.table) {
207
207
  throw new StorageError("Table not initialized");
@@ -217,12 +217,21 @@ export class LanceDBClient {
217
217
  // ignore
218
218
  }
219
219
  }
220
+ // Apply refineFactor for improved ANN recall (fetches N×candidates, reranks)
221
+ if (refineFactor && typeof refineFactor === "number") {
222
+ try {
223
+ query = query.refineFactor(refineFactor);
224
+ }
225
+ catch (_e) {
226
+ // ignore if not supported
227
+ }
228
+ }
220
229
  // Apply filter if provided
221
230
  if (filter) {
222
231
  query = query.where(filter);
223
232
  }
224
- // Execute search with limit
225
- const resultsArray = await query.limit(limit).toArray();
233
+ // Execute search with limit (and optional timeout)
234
+ const resultsArray = await query.limit(limit).toArray(timeoutMs ? { timeoutMs } : undefined);
226
235
  return resultsArray.map((row) => ({
227
236
  id: row.id,
228
237
  content: row.content,
@@ -385,6 +394,22 @@ export class LanceDBClient {
385
394
  };
386
395
  });
387
396
  }
397
+ /**
398
+ * Compact old data files and prune versions older than 7 days.
399
+ * Best-effort — never throws.
400
+ */
401
+ async optimize() {
402
+ if (!this.isConnected || !this.table)
403
+ return;
404
+ try {
405
+ await this.table.optimize({
406
+ cleanupOlderThan: new Date(Date.now() - 7 * 24 * 60 * 60 * 1000),
407
+ });
408
+ }
409
+ catch (_e) {
410
+ // Best-effort — never block normal operations
411
+ }
412
+ }
388
413
  /**
389
414
  * Sanitize an ID to prevent SQL injection
390
415
  * Removes any characters that aren't alphanumeric, underscore, or hyphen
@@ -201,7 +201,7 @@ export class LanceDBClient {
201
201
  await this.connect();
202
202
  }
203
203
  this._validateVector(vector);
204
- const { limit = 10, nprobes = 20, filter = null } = options;
204
+ const { limit = 10, nprobes = 20, filter = null, refineFactor, timeoutMs } = options;
205
205
  return this._retryOperation(async () => {
206
206
  if (!this.table) {
207
207
  throw new StorageError("Table not initialized");
@@ -217,12 +217,23 @@ export class LanceDBClient {
217
217
  // ignore
218
218
  }
219
219
  }
220
+ // Apply refineFactor for improved ANN recall (fetches N×candidates, reranks)
221
+ if (refineFactor && typeof refineFactor === "number") {
222
+ try {
223
+ query = query.refineFactor(refineFactor);
224
+ }
225
+ catch (_e) {
226
+ // ignore if not supported
227
+ }
228
+ }
220
229
  // Apply filter if provided
221
230
  if (filter) {
222
231
  query = query.where(filter);
223
232
  }
224
- // Execute search with limit
225
- const resultsArray = await query.limit(limit).toArray();
233
+ // Execute search with limit (and optional timeout)
234
+ const resultsArray = await query.limit(limit).toArray(
235
+ timeoutMs ? { timeoutMs } : undefined,
236
+ );
226
237
  return resultsArray.map((row) => ({
227
238
  id: row.id,
228
239
  content: row.content,
@@ -385,6 +396,21 @@ export class LanceDBClient {
385
396
  };
386
397
  });
387
398
  }
399
+ /**
400
+ * Compact old data files and prune versions older than 7 days.
401
+ * Best-effort — never throws.
402
+ */
403
+ async optimize() {
404
+ if (!this.isConnected || !this.table) return;
405
+ try {
406
+ await this.table.optimize({
407
+ cleanupOlderThan: new Date(Date.now() - 7 * 24 * 60 * 60 * 1000),
408
+ });
409
+ }
410
+ catch (_e) {
411
+ // Best-effort — never block normal operations
412
+ }
413
+ }
388
414
  /**
389
415
  * Sanitize an ID to prevent SQL injection
390
416
  * Removes any characters that aren't alphanumeric, underscore, or hyphen
@@ -225,6 +225,12 @@ export declare class MemoryMesh {
225
225
  reliability: any;
226
226
  use_count: any;
227
227
  }>;
228
+ /**
229
+ * Get a single synthesized skill by ID
230
+ * @param {string} id - Skill ID
231
+ * @returns {Promise<Object|null>} Skill data or null if not found
232
+ */
233
+ getSkill(id: any): Promise<any>;
228
234
  /**
229
235
  * Prune skills
230
236
  */
@@ -444,6 +450,11 @@ export declare class MemoryMesh {
444
450
  * await mesh.close(); // Clean up
445
451
  * ```
446
452
  */
453
+ /**
454
+ * Compact old data files and prune versions older than 7 days.
455
+ * Best-effort — delegates to LanceDBClient.optimize().
456
+ */
457
+ optimize(): Promise<any>;
447
458
  close(): Promise<void>;
448
459
  }
449
460
  /**
@@ -616,12 +616,30 @@ export class MemoryMesh {
616
616
  await this.init();
617
617
  const topic = options.topic || "general_improvement";
618
618
  const enrichedPrompt = options.enrichedPrompt || topic; // PHASE 4: Use enriched prompt
619
+ const mode = options.mode || "create";
620
+ const targetSkillId = options.targetSkillId;
619
621
  // const lookback = options.lookback || 20;
620
- logger.info({ topic, enrichedPrompt }, "Synthesizing logic");
622
+ logger.info({ topic, mode, targetSkillId }, "Synthesizing logic");
621
623
  // OPTIMIZATION: If we have an execution engine (kernel), use SkillCreator!
622
624
  if (this._kernel_execute) {
623
625
  logger.info("Dispatching to SkillCreator agent...");
624
626
  try {
627
+ // Fetch target skill content if refactoring
628
+ let targetContent = "";
629
+ let targetPath = "";
630
+ if (mode === "refactor" && targetSkillId) {
631
+ const skill = await this.getSkill(targetSkillId);
632
+ if (skill) {
633
+ targetPath = skill.metadata?.source_file || "";
634
+ if (targetPath && fs.existsSync(targetPath)) {
635
+ targetContent = fs.readFileSync(targetPath, "utf8");
636
+ }
637
+ else {
638
+ // Fallback to DB content if file missing
639
+ targetContent = skill.yamo_text || "";
640
+ }
641
+ }
642
+ }
625
643
  // Use stored skill directories
626
644
  const skillDirs = this.skillDirectories;
627
645
  // Track existing .yamo files before SkillCreator runs
@@ -652,7 +670,11 @@ export class MemoryMesh {
652
670
  }
653
671
  }
654
672
  // PHASE 4: Use enriched prompt for SkillCreator
655
- await this._kernel_execute(`SkillCreator: design a new skill to handle ${enrichedPrompt}`, {
673
+ let prompt = `SkillCreator: design a new skill to handle ${enrichedPrompt}`;
674
+ if (mode === "refactor" && targetContent) {
675
+ prompt = `SkillCreator: REFACTOR and FIX the following skill. It failed with the following context: ${enrichedPrompt}.\n\nEXISTING SKILL CONTENT:\n${targetContent}`;
676
+ }
677
+ await this._kernel_execute(prompt, {
656
678
  v1_1_enabled: true,
657
679
  });
658
680
  // Find newly created .yamo file
@@ -813,6 +835,37 @@ description: Auto-generated skill to handle: ${enrichedPrompt || topic}
813
835
  throw new Error(`Failed to update skill reliability: ${error.message}`);
814
836
  }
815
837
  }
838
+ /**
839
+ * Get a single synthesized skill by ID
840
+ * @param {string} id - Skill ID
841
+ * @returns {Promise<Object|null>} Skill data or null if not found
842
+ */
843
+ async getSkill(id) {
844
+ await this.init();
845
+ if (!this.skillTable) {
846
+ return null;
847
+ }
848
+ try {
849
+ const results = await this.skillTable
850
+ .query()
851
+ .filter(`id == '${id}'`)
852
+ .toArray();
853
+ if (results.length === 0) {
854
+ return null;
855
+ }
856
+ const record = results[0];
857
+ return {
858
+ ...record,
859
+ metadata: typeof record.metadata === "string"
860
+ ? JSON.parse(record.metadata)
861
+ : record.metadata,
862
+ };
863
+ }
864
+ catch (error) {
865
+ logger.warn({ err: error, id }, "Failed to get skill");
866
+ return null;
867
+ }
868
+ }
816
869
  /**
817
870
  * Prune skills
818
871
  */
@@ -1366,7 +1419,7 @@ description: Auto-generated skill to handle: ${enrichedPrompt || topic}
1366
1419
  `agent: MemoryMesh_${this.agentId};`,
1367
1420
  `intent: distill_wisdom_from_execution;`,
1368
1421
  `context:`,
1369
- ` original_context;${situation.replace(/;/g, ",")};`,
1422
+ ` original_context;${situation.replace(/;/g, "%3B")};`,
1370
1423
  ` error_pattern;${patternId};`,
1371
1424
  ` severity;${severity};`,
1372
1425
  ` timestamp;${timestamp};`,
@@ -1376,13 +1429,13 @@ description: Auto-generated skill to handle: ${enrichedPrompt || topic}
1376
1429
  `priority: high;`,
1377
1430
  `output:`,
1378
1431
  ` lesson_id;${lessonId};`,
1379
- ` oversight_description;${oversight.replace(/;/g, ",")};`,
1380
- ` preventative_rule;${preventativeRule.replace(/;/g, ",")};`,
1432
+ ` oversight_description;${oversight.replace(/;/g, "%3B")};`,
1433
+ ` preventative_rule;${preventativeRule.replace(/;/g, "%3B")};`,
1381
1434
  ` rule_confidence;${confidence};`,
1382
1435
  `meta:`,
1383
- ` rationale;${fix.replace(/;/g, ",")};`,
1384
- ` applicability_scope;${applicableScope.replace(/;/g, ",")};`,
1385
- ` inverse_lesson;${inverseLesson.replace(/;/g, ",")};`,
1436
+ ` rationale;${fix.replace(/;/g, "%3B")};`,
1437
+ ` applicability_scope;${applicableScope.replace(/;/g, "%3B")};`,
1438
+ ` inverse_lesson;${inverseLesson.replace(/;/g, "%3B")};`,
1386
1439
  ` confidence;${confidence};`,
1387
1440
  `log: lesson_learned;timestamp;${timestamp};pattern;${patternId};severity;${severity};id;${lessonId};`,
1388
1441
  `handoff: SubconsciousReflector;`,
@@ -1714,6 +1767,13 @@ description: Auto-generated skill to handle: ${enrichedPrompt || topic}
1714
1767
  * await mesh.close(); // Clean up
1715
1768
  * ```
1716
1769
  */
1770
+ /**
1771
+ * Compact old data files and prune versions older than 7 days.
1772
+ * Best-effort — delegates to LanceDBClient.optimize().
1773
+ */
1774
+ async optimize() {
1775
+ return this.client?.optimize?.();
1776
+ }
1717
1777
  // eslint-disable-next-line @typescript-eslint/require-await
1718
1778
  async close() {
1719
1779
  try {
@@ -648,12 +648,32 @@ export class MemoryMesh {
648
648
  await this.init();
649
649
  const topic = options.topic || "general_improvement";
650
650
  const enrichedPrompt = options.enrichedPrompt || topic; // PHASE 4: Use enriched prompt
651
+ const mode = options.mode || "create";
652
+ const targetSkillId = options.targetSkillId;
653
+
651
654
  // const lookback = options.lookback || 20;
652
- logger.info({ topic, enrichedPrompt }, "Synthesizing logic");
655
+ logger.info({ topic, mode, targetSkillId }, "Synthesizing logic");
656
+
653
657
  // OPTIMIZATION: If we have an execution engine (kernel), use SkillCreator!
654
658
  if (this._kernel_execute) {
655
659
  logger.info("Dispatching to SkillCreator agent...");
656
660
  try {
661
+ // Fetch target skill content if refactoring
662
+ let targetContent = "";
663
+ let targetPath = "";
664
+ if (mode === "refactor" && targetSkillId) {
665
+ const skill = await this.getSkill(targetSkillId);
666
+ if (skill) {
667
+ targetPath = skill.metadata?.source_file || "";
668
+ if (targetPath && fs.existsSync(targetPath)) {
669
+ targetContent = fs.readFileSync(targetPath, "utf8");
670
+ } else {
671
+ // Fallback to DB content if file missing
672
+ targetContent = skill.yamo_text || "";
673
+ }
674
+ }
675
+ }
676
+
657
677
  // Use stored skill directories
658
678
  const skillDirs = this.skillDirectories;
659
679
  // Track existing .yamo files before SkillCreator runs
@@ -683,8 +703,14 @@ export class MemoryMesh {
683
703
  walk(dir);
684
704
  }
685
705
  }
706
+
686
707
  // PHASE 4: Use enriched prompt for SkillCreator
687
- await this._kernel_execute(`SkillCreator: design a new skill to handle ${enrichedPrompt}`, {
708
+ let prompt = `SkillCreator: design a new skill to handle ${enrichedPrompt}`;
709
+ if (mode === "refactor" && targetContent) {
710
+ prompt = `SkillCreator: REFACTOR and FIX the following skill. It failed with the following context: ${enrichedPrompt}.\n\nEXISTING SKILL CONTENT:\n${targetContent}`;
711
+ }
712
+
713
+ await this._kernel_execute(prompt, {
688
714
  v1_1_enabled: true,
689
715
  });
690
716
  // Find newly created .yamo file
@@ -845,6 +871,37 @@ description: Auto-generated skill to handle: ${enrichedPrompt || topic}
845
871
  throw new Error(`Failed to update skill reliability: ${error.message}`);
846
872
  }
847
873
  }
874
+ /**
875
+ * Get a single synthesized skill by ID
876
+ * @param {string} id - Skill ID
877
+ * @returns {Promise<Object|null>} Skill data or null if not found
878
+ */
879
+ async getSkill(id) {
880
+ await this.init();
881
+ if (!this.skillTable) {
882
+ return null;
883
+ }
884
+ try {
885
+ const results = await this.skillTable
886
+ .query()
887
+ .filter(`id == '${id}'`)
888
+ .toArray();
889
+ if (results.length === 0) {
890
+ return null;
891
+ }
892
+ const record = results[0];
893
+ return {
894
+ ...record,
895
+ metadata: typeof record.metadata === "string"
896
+ ? JSON.parse(record.metadata)
897
+ : record.metadata,
898
+ };
899
+ }
900
+ catch (error) {
901
+ logger.warn({ err: error, id }, "Failed to get skill");
902
+ return null;
903
+ }
904
+ }
848
905
  /**
849
906
  * Prune skills
850
907
  */
@@ -1418,7 +1475,7 @@ description: Auto-generated skill to handle: ${enrichedPrompt || topic}
1418
1475
  `agent: MemoryMesh_${this.agentId};`,
1419
1476
  `intent: distill_wisdom_from_execution;`,
1420
1477
  `context:`,
1421
- ` original_context;${situation.replace(/;/g, ",")};`,
1478
+ ` original_context;${situation.replace(/;/g, "%3B")};`,
1422
1479
  ` error_pattern;${patternId};`,
1423
1480
  ` severity;${severity};`,
1424
1481
  ` timestamp;${timestamp};`,
@@ -1428,13 +1485,13 @@ description: Auto-generated skill to handle: ${enrichedPrompt || topic}
1428
1485
  `priority: high;`,
1429
1486
  `output:`,
1430
1487
  ` lesson_id;${lessonId};`,
1431
- ` oversight_description;${oversight.replace(/;/g, ",")};`,
1432
- ` preventative_rule;${preventativeRule.replace(/;/g, ",")};`,
1488
+ ` oversight_description;${oversight.replace(/;/g, "%3B")};`,
1489
+ ` preventative_rule;${preventativeRule.replace(/;/g, "%3B")};`,
1433
1490
  ` rule_confidence;${confidence};`,
1434
1491
  `meta:`,
1435
- ` rationale;${fix.replace(/;/g, ",")};`,
1436
- ` applicability_scope;${applicableScope.replace(/;/g, ",")};`,
1437
- ` inverse_lesson;${inverseLesson.replace(/;/g, ",")};`,
1492
+ ` rationale;${fix.replace(/;/g, "%3B")};`,
1493
+ ` applicability_scope;${applicableScope.replace(/;/g, "%3B")};`,
1494
+ ` inverse_lesson;${inverseLesson.replace(/;/g, "%3B")};`,
1438
1495
  ` confidence;${confidence};`,
1439
1496
  `log: lesson_learned;timestamp;${timestamp};pattern;${patternId};severity;${severity};id;${lessonId};`,
1440
1497
  `handoff: SubconsciousReflector;`,
@@ -1801,6 +1858,13 @@ description: Auto-generated skill to handle: ${enrichedPrompt || topic}
1801
1858
  * await mesh.close(); // Clean up
1802
1859
  * ```
1803
1860
  */
1861
+ /**
1862
+ * Compact old data files and prune versions older than 7 days.
1863
+ * Best-effort — delegates to LanceDBClient.optimize().
1864
+ */
1865
+ async optimize() {
1866
+ return this.client?.optimize?.();
1867
+ }
1804
1868
  // eslint-disable-next-line @typescript-eslint/require-await
1805
1869
  async close() {
1806
1870
  try {
@@ -65,6 +65,12 @@ export declare function isSchemaV2(schema: any): any;
65
65
  * Safe to call on any table — non-memory tables skip the schema column additions.
66
66
  */
67
67
  export declare function migrateTableV2(table: any): Promise<void>;
68
+ /**
69
+ * Ensure the vector column has an IVF_PQ index.
70
+ * Skipped when: table has too few rows, index already exists, or table is a mock.
71
+ * Called automatically by createMemoryTableWithDimension after migration.
72
+ */
73
+ export declare function ensureVectorIndex(table: any): Promise<void>;
68
74
  /**
69
75
  * Memory table schema using Apache Arrow format (default 384 dimensions)
70
76
  * @deprecated Use createMemorySchema(vectorDim) for dynamic dimensions
@@ -122,6 +128,7 @@ declare const _default: {
122
128
  createMemorySchemaV2: typeof createMemorySchemaV2;
123
129
  isSchemaV2: typeof isSchemaV2;
124
130
  migrateTableV2: typeof migrateTableV2;
131
+ ensureVectorIndex: typeof ensureVectorIndex;
125
132
  getEmbeddingDimension: typeof getEmbeddingDimension;
126
133
  DEFAULT_VECTOR_DIMENSION: number;
127
134
  EMBEDDING_DIMENSIONS: {
@@ -9,6 +9,7 @@
9
9
  * - text-embedding-3-small: 1536 dimensions
10
10
  */
11
11
  import * as arrow from "apache-arrow";
12
+ import { Index } from "@lancedb/lancedb";
12
13
  /**
13
14
  * Default vector dimension (all-MiniLM-L6-v2)
14
15
  */
@@ -153,6 +154,34 @@ export async function migrateTableV2(table) {
153
154
  { name: "last_accessed", valueSql: "cast(null as timestamp)" },
154
155
  ]);
155
156
  }
157
+ /**
158
+ * Ensure the vector column has an IVF_PQ index.
159
+ * Skipped when: table has too few rows, index already exists, or table is a mock.
160
+ * Called automatically by createMemoryTableWithDimension after migration.
161
+ */
162
+ export async function ensureVectorIndex(table) {
163
+ if (typeof table.listIndices !== "function")
164
+ return;
165
+ try {
166
+ const indices = await table.listIndices();
167
+ if (indices.some((i) => i.columns.includes("vector")))
168
+ return;
169
+ const rowCount = await table.countRows();
170
+ if (rowCount < INDEX_CONFIG.vector.num_partitions)
171
+ return;
172
+ await table.createIndex("vector", {
173
+ config: Index.ivfPq({
174
+ numPartitions: INDEX_CONFIG.vector.num_partitions,
175
+ numSubVectors: INDEX_CONFIG.vector.num_sub_vectors,
176
+ distanceType: INDEX_CONFIG.vector.metric,
177
+ }),
178
+ replace: false,
179
+ });
180
+ }
181
+ catch {
182
+ // Index creation is best-effort — never block table access
183
+ }
184
+ }
156
185
  /**
157
186
  * Memory table schema using Apache Arrow format (default 384 dimensions)
158
187
  * @deprecated Use createMemorySchema(vectorDim) for dynamic dimensions
@@ -209,6 +238,8 @@ export async function createMemoryTableWithDimension(db, tableName, vectorDim) {
209
238
  }
210
239
  // Migrate existing tables to V2 (manifest paths + schema columns, idempotent)
211
240
  await migrateTableV2(table);
241
+ // Ensure vector index exists (no-op if already present or insufficient rows)
242
+ await ensureVectorIndex(table);
212
243
  return table;
213
244
  }
214
245
  catch (error) {
@@ -225,6 +256,7 @@ export default {
225
256
  createMemorySchemaV2,
226
257
  isSchemaV2,
227
258
  migrateTableV2,
259
+ ensureVectorIndex,
228
260
  getEmbeddingDimension,
229
261
  DEFAULT_VECTOR_DIMENSION,
230
262
  EMBEDDING_DIMENSIONS,
@@ -9,6 +9,8 @@
9
9
  * - text-embedding-3-small: 1536 dimensions
10
10
  */
11
11
  import * as arrow from "apache-arrow";
12
+ import * as lancedb from "@lancedb/lancedb";
13
+ import { Index } from "@lancedb/lancedb";
12
14
  /**
13
15
  * Default vector dimension (all-MiniLM-L6-v2)
14
16
  */
@@ -150,6 +152,31 @@ export async function migrateTableV2(table) {
150
152
  { name: "last_accessed", valueSql: "cast(null as timestamp)" },
151
153
  ]);
152
154
  }
155
+ /**
156
+ * Ensure the vector column has an IVF_PQ index.
157
+ * Skipped when: table has too few rows, index already exists, or table is a mock.
158
+ * Called automatically by createMemoryTableWithDimension after migration.
159
+ */
160
+ export async function ensureVectorIndex(table) {
161
+ if (typeof table.listIndices !== "function") return;
162
+ try {
163
+ const indices = await table.listIndices();
164
+ if (indices.some((i) => i.columns.includes("vector"))) return;
165
+ const rowCount = await table.countRows();
166
+ if (rowCount < INDEX_CONFIG.vector.num_partitions) return;
167
+ await table.createIndex("vector", {
168
+ config: Index.ivfPq({
169
+ numPartitions: INDEX_CONFIG.vector.num_partitions,
170
+ numSubVectors: INDEX_CONFIG.vector.num_sub_vectors,
171
+ distanceType: INDEX_CONFIG.vector.metric,
172
+ }),
173
+ replace: false,
174
+ });
175
+ }
176
+ catch {
177
+ // Index creation is best-effort — never block table access
178
+ }
179
+ }
153
180
  /**
154
181
  * Memory table schema using Apache Arrow format (default 384 dimensions)
155
182
  * @deprecated Use createMemorySchema(vectorDim) for dynamic dimensions
@@ -206,6 +233,8 @@ export async function createMemoryTableWithDimension(db, tableName, vectorDim) {
206
233
  }
207
234
  // Migrate existing tables to V2 (manifest paths + schema columns, idempotent)
208
235
  await migrateTableV2(table);
236
+ // Ensure vector index exists (no-op if already present or insufficient rows)
237
+ await ensureVectorIndex(table);
209
238
  return table;
210
239
  }
211
240
  catch (error) {
@@ -222,6 +251,7 @@ export default {
222
251
  createMemorySchemaV2,
223
252
  isSchemaV2,
224
253
  migrateTableV2,
254
+ ensureVectorIndex,
225
255
  getEmbeddingDimension,
226
256
  DEFAULT_VECTOR_DIMENSION,
227
257
  EMBEDDING_DIMENSIONS,
@@ -0,0 +1,32 @@
1
+ /**
2
+ * Prompt Security Utilities — RFC-0010-A
3
+ *
4
+ * Shared sanitisation functions for all LLM prompt construction.
5
+ *
6
+ * Rule (RFC-0010-A §5): Any string originating from user input, memory retrieval,
7
+ * or execution logs that is interpolated into an LLM prompt MUST pass through
8
+ * sanitizePromptField() before interpolation.
9
+ *
10
+ * Audit sweep (2026-03-14):
11
+ * - lib/memory/memory-mesh.ts — HyDE query expansion uses `scrubbed` (pre-sanitised)
12
+ * - lib/memory/memory-mesh.ts — synthesis prompt uses `scrubbed` (pre-sanitised)
13
+ * - lib/llm/client.ts — reflection prompt: static system + formatted memory list
14
+ * - lib/memory/memory-translator.ts — role-confusion protection already applied
15
+ */
16
+ /**
17
+ * Sanitise a free-text value before injecting it into an LLM prompt.
18
+ *
19
+ * Actions:
20
+ * - Collapses \r\n and \n to a single space (prevents instruction-injection via newlines)
21
+ * - Escapes double-quotes → \" (prevents breaking out of JSON-encoded fields)
22
+ * - Caps at maxLen characters (prevents token-budget exhaustion)
23
+ */
24
+ export declare function sanitizePromptField(value: string, maxLen?: number): string;
25
+ /**
26
+ * Sanitise a memory or agent identifier before injecting it into an LLM prompt.
27
+ *
28
+ * Allows only alphanumeric characters, underscores, and hyphens.
29
+ * Strips everything else to prevent SQL-style keywords, path traversal, or quote
30
+ * characters from appearing in the prompt even if the ID was tampered with in storage.
31
+ */
32
+ export declare function sanitizeSkillId(value: string, maxLen?: number): string;
@@ -0,0 +1,43 @@
1
+ /**
2
+ * Prompt Security Utilities — RFC-0010-A
3
+ *
4
+ * Shared sanitisation functions for all LLM prompt construction.
5
+ *
6
+ * Rule (RFC-0010-A §5): Any string originating from user input, memory retrieval,
7
+ * or execution logs that is interpolated into an LLM prompt MUST pass through
8
+ * sanitizePromptField() before interpolation.
9
+ *
10
+ * Audit sweep (2026-03-14):
11
+ * - lib/memory/memory-mesh.ts — HyDE query expansion uses `scrubbed` (pre-sanitised)
12
+ * - lib/memory/memory-mesh.ts — synthesis prompt uses `scrubbed` (pre-sanitised)
13
+ * - lib/llm/client.ts — reflection prompt: static system + formatted memory list
14
+ * - lib/memory/memory-translator.ts — role-confusion protection already applied
15
+ */
16
+ /**
17
+ * Sanitise a free-text value before injecting it into an LLM prompt.
18
+ *
19
+ * Actions:
20
+ * - Collapses \r\n and \n to a single space (prevents instruction-injection via newlines)
21
+ * - Escapes double-quotes → \" (prevents breaking out of JSON-encoded fields)
22
+ * - Caps at maxLen characters (prevents token-budget exhaustion)
23
+ */
24
+ export function sanitizePromptField(value, maxLen = 256) {
25
+ if (!value)
26
+ return "";
27
+ return value
28
+ .replace(/[\r\n]+/g, " ")
29
+ .replace(/"/g, '\\"')
30
+ .slice(0, maxLen);
31
+ }
32
+ /**
33
+ * Sanitise a memory or agent identifier before injecting it into an LLM prompt.
34
+ *
35
+ * Allows only alphanumeric characters, underscores, and hyphens.
36
+ * Strips everything else to prevent SQL-style keywords, path traversal, or quote
37
+ * characters from appearing in the prompt even if the ID was tampered with in storage.
38
+ */
39
+ export function sanitizeSkillId(value, maxLen = 64) {
40
+ if (!value)
41
+ return "";
42
+ return value.replace(/[^a-zA-Z0-9_-]/g, "").slice(0, maxLen);
43
+ }
@@ -0,0 +1,43 @@
1
+ /**
2
+ * Prompt Security Utilities — RFC-0010-A
3
+ *
4
+ * Shared sanitisation functions for all LLM prompt construction.
5
+ *
6
+ * Rule (RFC-0010-A §5): Any string originating from user input, memory retrieval,
7
+ * or execution logs that is interpolated into an LLM prompt MUST pass through
8
+ * sanitizePromptField() before interpolation.
9
+ *
10
+ * Audit sweep (2026-03-14):
11
+ * - lib/memory/memory-mesh.ts — HyDE query expansion uses `scrubbed` (pre-sanitised)
12
+ * - lib/memory/memory-mesh.ts — synthesis prompt uses `scrubbed` (pre-sanitised)
13
+ * - lib/llm/client.ts — reflection prompt: static system + formatted memory list
14
+ * - lib/memory/memory-translator.ts — role-confusion protection already applied
15
+ */
16
+
17
+ /**
18
+ * Sanitise a free-text value before injecting it into an LLM prompt.
19
+ *
20
+ * Actions:
21
+ * - Collapses \r\n and \n to a single space (prevents instruction-injection via newlines)
22
+ * - Escapes double-quotes → \" (prevents breaking out of JSON-encoded fields)
23
+ * - Caps at maxLen characters (prevents token-budget exhaustion)
24
+ */
25
+ export function sanitizePromptField(value: string, maxLen = 256): string {
26
+ if (!value) return "";
27
+ return value
28
+ .replace(/[\r\n]+/g, " ")
29
+ .replace(/"/g, '\\"')
30
+ .slice(0, maxLen);
31
+ }
32
+
33
+ /**
34
+ * Sanitise a memory or agent identifier before injecting it into an LLM prompt.
35
+ *
36
+ * Allows only alphanumeric characters, underscores, and hyphens.
37
+ * Strips everything else to prevent SQL-style keywords, path traversal, or quote
38
+ * characters from appearing in the prompt even if the ID was tampered with in storage.
39
+ */
40
+ export function sanitizeSkillId(value: string, maxLen = 64): string {
41
+ if (!value) return "";
42
+ return value.replace(/[^a-zA-Z0-9_-]/g, "").slice(0, maxLen);
43
+ }
@@ -1,12 +1,13 @@
1
1
  /**
2
- * YAMO Emitter - Constructs structured YAMO blocks for auditability
2
+ * YAMO Emitter - Constructs structured YAMO ABNF blocks for auditability
3
3
  *
4
- * Based on YAMO Protocol specification:
5
- * - Semicolon-terminated key-value pairs
6
- * - Agent/Intent/Context/Constraints/Meta/Output structure
7
- * - Supports reflect, retain, recall operations
4
+ * Based on YAMO Protocol RFC-0011 §3.2 (ABNF colon-separated multi-line format).
5
+ * This format is DISTINCT from the flat wire format (RFC-0008 / RFC-0014).
8
6
  *
9
- * Reference: Hindsight project's yamo_integration.py
7
+ * Escaping: RFC-0014 semicolons in values are percent-encoded as `%3B`.
8
+ * (Supersedes the comma-escape scheme used prior to 2026-03-14.)
9
+ *
10
+ * Reference: yamo-os lib/yamo/emitter.ts (canonical implementation)
10
11
  */
11
12
  /**
12
13
  * YamoEmitter class for building YAMO protocol blocks
@@ -1,14 +1,19 @@
1
1
  // @ts-nocheck
2
2
  /**
3
- * YAMO Emitter - Constructs structured YAMO blocks for auditability
3
+ * YAMO Emitter - Constructs structured YAMO ABNF blocks for auditability
4
4
  *
5
- * Based on YAMO Protocol specification:
6
- * - Semicolon-terminated key-value pairs
7
- * - Agent/Intent/Context/Constraints/Meta/Output structure
8
- * - Supports reflect, retain, recall operations
5
+ * Based on YAMO Protocol RFC-0011 §3.2 (ABNF colon-separated multi-line format).
6
+ * This format is DISTINCT from the flat wire format (RFC-0008 / RFC-0014).
9
7
  *
10
- * Reference: Hindsight project's yamo_integration.py
8
+ * Escaping: RFC-0014 semicolons in values are percent-encoded as `%3B`.
9
+ * (Supersedes the comma-escape scheme used prior to 2026-03-14.)
10
+ *
11
+ * Reference: yamo-os lib/yamo/emitter.ts (canonical implementation)
11
12
  */
13
+ /** Percent-encode semicolons in a value so they don't break the YAMO line format. */
14
+ function escapeValue(value) {
15
+ return String(value).replace(/;/g, "%3B");
16
+ }
12
17
  /**
13
18
  * YamoEmitter class for building YAMO protocol blocks
14
19
  * YAMO (Yet Another Multi-agent Orchestration) blocks provide
@@ -22,17 +27,17 @@ export class YamoEmitter {
22
27
  static buildReflectBlock(params) {
23
28
  const { topic, memoryCount, agentId = "default", reflection, confidence = 0.8, } = params;
24
29
  const timestamp = new Date().toISOString();
25
- return `agent: MemoryMesh_${agentId};
30
+ return `agent: MemoryMesh_${escapeValue(agentId)};
26
31
  intent: synthesize_insights_from_context;
27
32
  context:
28
- topic;${topic || "general"};
33
+ topic;${escapeValue(topic || "general")};
29
34
  memory_count;${memoryCount};
30
35
  timestamp;${timestamp};
31
36
  constraints:
32
37
  hypothesis;Reflection generates new insights from existing facts;
33
38
  priority: high;
34
39
  output:
35
- reflection;${reflection};
40
+ reflection;${escapeValue(reflection)};
36
41
  confidence;${confidence};
37
42
  meta:
38
43
  rationale;Synthesized from ${memoryCount} relevant memories;
@@ -50,26 +55,26 @@ handoff: End;
50
55
  const { content, metadata: _metadata = {}, id, agentId = "default", memoryType = "event", } = params;
51
56
  const timestamp = new Date().toISOString();
52
57
  const contentPreview = content.length > 100 ? `${content.substring(0, 100)}...` : content;
53
- // Escape semicolons in content for YAMO format
54
- const escapedContent = contentPreview.replace(/;/g, ",");
55
- return `agent: MemoryMesh_${agentId};
58
+ // RFC-0014: percent-encode semicolons (replaces legacy comma-escape)
59
+ const escapedContent = escapeValue(contentPreview);
60
+ return `agent: MemoryMesh_${escapeValue(agentId)};
56
61
  intent: store_memory_for_future_retrieval;
57
62
  context:
58
- memory_id;${id};
59
- memory_type;${memoryType};
63
+ memory_id;${escapeValue(id)};
64
+ memory_type;${escapeValue(memoryType)};
60
65
  timestamp;${timestamp};
61
66
  content_length;${content.length};
62
67
  constraints:
63
68
  hypothesis;New information should be integrated into world model;
64
69
  priority: medium;
65
70
  output:
66
- memory_stored;${id};
71
+ memory_stored;${escapeValue(id)};
67
72
  content_preview;${escapedContent};
68
73
  meta:
69
74
  rationale;Memory persisted for semantic search and retrieval;
70
75
  observation;Content vectorized and stored in LanceDB;
71
76
  confidence;1.0;
72
- log: memory_retained;timestamp;${timestamp};id;${id};type;${memoryType};
77
+ log: memory_retained;timestamp;${timestamp};id;${escapeValue(id)};type;${escapeValue(memoryType)};
73
78
  handoff: End;
74
79
  `;
75
80
  }
@@ -81,11 +86,11 @@ handoff: End;
81
86
  const { query, resultCount, limit = 10, agentId = "default", searchType = "semantic", } = params;
82
87
  const timestamp = new Date().toISOString();
83
88
  const recallRatio = resultCount > 0 ? (resultCount / limit).toFixed(2) : "0.00";
84
- return `agent: MemoryMesh_${agentId};
89
+ return `agent: MemoryMesh_${escapeValue(agentId)};
85
90
  intent: retrieve_relevant_memories;
86
91
  context:
87
- query;${query};
88
- search_type;${searchType};
92
+ query;${escapeValue(query)};
93
+ search_type;${escapeValue(searchType)};
89
94
  requested_limit;${limit};
90
95
  timestamp;${timestamp};
91
96
  constraints:
@@ -98,7 +103,7 @@ meta:
98
103
  rationale;Semantic search finds similar content by vector similarity;
99
104
  observation;${resultCount} memories found matching query;
100
105
  confidence;${resultCount > 0 ? "0.9" : "0.5"};
101
- log: memory_recalled;timestamp;${timestamp};results;${resultCount};query;${query};
106
+ log: memory_recalled;timestamp;${timestamp};results;${resultCount};query;${escapeValue(query)};
102
107
  handoff: End;
103
108
  `;
104
109
  }
@@ -109,22 +114,22 @@ handoff: End;
109
114
  static buildDeleteBlock(params) {
110
115
  const { id, agentId = "default", reason = "user_request" } = params;
111
116
  const timestamp = new Date().toISOString();
112
- return `agent: MemoryMesh_${agentId};
117
+ return `agent: MemoryMesh_${escapeValue(agentId)};
113
118
  intent: remove_memory_from_storage;
114
119
  context:
115
- memory_id;${id};
116
- reason;${reason};
120
+ memory_id;${escapeValue(id)};
121
+ reason;${escapeValue(reason)};
117
122
  timestamp;${timestamp};
118
123
  constraints:
119
124
  hypothesis;Memory removal should be traceable for audit;
120
125
  priority: low;
121
126
  output:
122
- deleted;${id};
127
+ deleted;${escapeValue(id)};
123
128
  meta:
124
129
  rationale;Memory removed from vector store;
125
130
  observation;Deletion recorded for provenance;
126
131
  confidence;1.0;
127
- log: memory_deleted;timestamp;${timestamp};id;${id};
132
+ log: memory_deleted;timestamp;${timestamp};id;${escapeValue(id)};
128
133
  handoff: End;
129
134
  `;
130
135
  }
@@ -157,7 +162,8 @@ handoff: End;
157
162
  // Allow empty lines and comments
158
163
  if (trimmed &&
159
164
  !trimmed.startsWith("agent:") &&
160
- !trimmed.startsWith("handoff:")) {
165
+ !trimmed.startsWith("handoff:") &&
166
+ !trimmed.endsWith(":")) {
161
167
  errors.push(`Line not semicolon-terminated: ${trimmed.substring(0, 50)}`);
162
168
  }
163
169
  }
@@ -1,14 +1,20 @@
1
1
  // @ts-nocheck
2
2
  /**
3
- * YAMO Emitter - Constructs structured YAMO blocks for auditability
3
+ * YAMO Emitter - Constructs structured YAMO ABNF blocks for auditability
4
4
  *
5
- * Based on YAMO Protocol specification:
6
- * - Semicolon-terminated key-value pairs
7
- * - Agent/Intent/Context/Constraints/Meta/Output structure
8
- * - Supports reflect, retain, recall operations
5
+ * Based on YAMO Protocol RFC-0011 §3.2 (ABNF colon-separated multi-line format).
6
+ * This format is DISTINCT from the flat wire format (RFC-0008 / RFC-0014).
9
7
  *
10
- * Reference: Hindsight project's yamo_integration.py
8
+ * Escaping: RFC-0014 semicolons in values are percent-encoded as `%3B`.
9
+ * (Supersedes the comma-escape scheme used prior to 2026-03-14.)
10
+ *
11
+ * Reference: yamo-os lib/yamo/emitter.ts (canonical implementation)
11
12
  */
13
+
14
+ /** Percent-encode semicolons in a value so they don't break the YAMO line format. */
15
+ function escapeValue(value: string): string {
16
+ return String(value).replace(/;/g, "%3B");
17
+ }
12
18
  /**
13
19
  * YamoEmitter class for building YAMO protocol blocks
14
20
  * YAMO (Yet Another Multi-agent Orchestration) blocks provide
@@ -22,17 +28,17 @@ export class YamoEmitter {
22
28
  static buildReflectBlock(params) {
23
29
  const { topic, memoryCount, agentId = "default", reflection, confidence = 0.8, } = params;
24
30
  const timestamp = new Date().toISOString();
25
- return `agent: MemoryMesh_${agentId};
31
+ return `agent: MemoryMesh_${escapeValue(agentId)};
26
32
  intent: synthesize_insights_from_context;
27
33
  context:
28
- topic;${topic || "general"};
34
+ topic;${escapeValue(topic || "general")};
29
35
  memory_count;${memoryCount};
30
36
  timestamp;${timestamp};
31
37
  constraints:
32
38
  hypothesis;Reflection generates new insights from existing facts;
33
39
  priority: high;
34
40
  output:
35
- reflection;${reflection};
41
+ reflection;${escapeValue(reflection)};
36
42
  confidence;${confidence};
37
43
  meta:
38
44
  rationale;Synthesized from ${memoryCount} relevant memories;
@@ -50,26 +56,26 @@ handoff: End;
50
56
  const { content, metadata: _metadata = {}, id, agentId = "default", memoryType = "event", } = params;
51
57
  const timestamp = new Date().toISOString();
52
58
  const contentPreview = content.length > 100 ? `${content.substring(0, 100)}...` : content;
53
- // Escape semicolons in content for YAMO format
54
- const escapedContent = contentPreview.replace(/;/g, ",");
55
- return `agent: MemoryMesh_${agentId};
59
+ // RFC-0014: percent-encode semicolons (replaces legacy comma-escape)
60
+ const escapedContent = escapeValue(contentPreview);
61
+ return `agent: MemoryMesh_${escapeValue(agentId)};
56
62
  intent: store_memory_for_future_retrieval;
57
63
  context:
58
- memory_id;${id};
59
- memory_type;${memoryType};
64
+ memory_id;${escapeValue(id)};
65
+ memory_type;${escapeValue(memoryType)};
60
66
  timestamp;${timestamp};
61
67
  content_length;${content.length};
62
68
  constraints:
63
69
  hypothesis;New information should be integrated into world model;
64
70
  priority: medium;
65
71
  output:
66
- memory_stored;${id};
72
+ memory_stored;${escapeValue(id)};
67
73
  content_preview;${escapedContent};
68
74
  meta:
69
75
  rationale;Memory persisted for semantic search and retrieval;
70
76
  observation;Content vectorized and stored in LanceDB;
71
77
  confidence;1.0;
72
- log: memory_retained;timestamp;${timestamp};id;${id};type;${memoryType};
78
+ log: memory_retained;timestamp;${timestamp};id;${escapeValue(id)};type;${escapeValue(memoryType)};
73
79
  handoff: End;
74
80
  `;
75
81
  }
@@ -81,11 +87,11 @@ handoff: End;
81
87
  const { query, resultCount, limit = 10, agentId = "default", searchType = "semantic", } = params;
82
88
  const timestamp = new Date().toISOString();
83
89
  const recallRatio = resultCount > 0 ? (resultCount / limit).toFixed(2) : "0.00";
84
- return `agent: MemoryMesh_${agentId};
90
+ return `agent: MemoryMesh_${escapeValue(agentId)};
85
91
  intent: retrieve_relevant_memories;
86
92
  context:
87
- query;${query};
88
- search_type;${searchType};
93
+ query;${escapeValue(query)};
94
+ search_type;${escapeValue(searchType)};
89
95
  requested_limit;${limit};
90
96
  timestamp;${timestamp};
91
97
  constraints:
@@ -98,7 +104,7 @@ meta:
98
104
  rationale;Semantic search finds similar content by vector similarity;
99
105
  observation;${resultCount} memories found matching query;
100
106
  confidence;${resultCount > 0 ? "0.9" : "0.5"};
101
- log: memory_recalled;timestamp;${timestamp};results;${resultCount};query;${query};
107
+ log: memory_recalled;timestamp;${timestamp};results;${resultCount};query;${escapeValue(query)};
102
108
  handoff: End;
103
109
  `;
104
110
  }
@@ -109,22 +115,22 @@ handoff: End;
109
115
  static buildDeleteBlock(params) {
110
116
  const { id, agentId = "default", reason = "user_request" } = params;
111
117
  const timestamp = new Date().toISOString();
112
- return `agent: MemoryMesh_${agentId};
118
+ return `agent: MemoryMesh_${escapeValue(agentId)};
113
119
  intent: remove_memory_from_storage;
114
120
  context:
115
- memory_id;${id};
116
- reason;${reason};
121
+ memory_id;${escapeValue(id)};
122
+ reason;${escapeValue(reason)};
117
123
  timestamp;${timestamp};
118
124
  constraints:
119
125
  hypothesis;Memory removal should be traceable for audit;
120
126
  priority: low;
121
127
  output:
122
- deleted;${id};
128
+ deleted;${escapeValue(id)};
123
129
  meta:
124
130
  rationale;Memory removed from vector store;
125
131
  observation;Deletion recorded for provenance;
126
132
  confidence;1.0;
127
- log: memory_deleted;timestamp;${timestamp};id;${id};
133
+ log: memory_deleted;timestamp;${timestamp};id;${escapeValue(id)};
128
134
  handoff: End;
129
135
  `;
130
136
  }
@@ -157,7 +163,8 @@ handoff: End;
157
163
  // Allow empty lines and comments
158
164
  if (trimmed &&
159
165
  !trimmed.startsWith("agent:") &&
160
- !trimmed.startsWith("handoff:")) {
166
+ !trimmed.startsWith("handoff:") &&
167
+ !trimmed.endsWith(":")) {
161
168
  errors.push(`Line not semicolon-terminated: ${trimmed.substring(0, 50)}`);
162
169
  }
163
170
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@yamo/memory-mesh",
3
- "version": "3.2.4",
3
+ "version": "3.2.6",
4
4
  "description": "Portable semantic memory system with Layer 0 Scrubber for YAMO agents (v3 Singularity Edition)",
5
5
  "repository": {
6
6
  "type": "git",