@sentry/junior-memory 0.131.0 → 0.133.0

package/migrations/0006_friendly_hydra.sql

@@ -0,0 +1,4 @@
+CREATE EXTENSION IF NOT EXISTS btree_gin;--> statement-breakpoint
+DROP INDEX "junior_memory_memories_search_idx";--> statement-breakpoint
+ALTER TABLE "junior_memory_memories" ADD COLUMN "search_vector" "tsvector" GENERATED ALWAYS AS (to_tsvector('english', "content")) STORED;--> statement-breakpoint
+CREATE INDEX "junior_memory_memories_search_idx" ON "junior_memory_memories" USING gin ("scope","scope_key","search_vector") WHERE "junior_memory_memories"."archived_at_ms" IS NULL AND "junior_memory_memories"."superseded_at_ms" IS NULL AND "junior_memory_memories"."superseded_by_id" IS NULL;

package/migrations/meta/0006_snapshot.json

@@ -0,0 +1,386 @@
+{
+  "id": "f18fa309-6485-40f3-921a-5d65552fbdf6",
+  "prevId": "6948ea3f-2e44-49ce-aa94-8a993ce61cec",
+  "version": "7",
+  "dialect": "postgresql",
+  "tables": {
+    "public.junior_memory_embeddings": {
+      "name": "junior_memory_embeddings",
+      "schema": "",
+      "columns": {
+        "memory_id": {
+          "name": "memory_id",
+          "type": "text",
+          "primaryKey": true,
+          "notNull": true
+        },
+        "provider": {
+          "name": "provider",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "model": {
+          "name": "model",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "dimensions": {
+          "name": "dimensions",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "metric": {
+          "name": "metric",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "content_hash": {
+          "name": "content_hash",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "embedding": {
+          "name": "embedding",
+          "type": "vector(1536)",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "created_at_ms": {
+          "name": "created_at_ms",
+          "type": "bigint",
+          "primaryKey": false,
+          "notNull": true
+        }
+      },
+      "indexes": {
+        "junior_memory_embeddings_model_idx": {
+          "name": "junior_memory_embeddings_model_idx",
+          "columns": [
+            {
+              "expression": "provider",
+              "isExpression": false,
+              "asc": true,
+              "nulls": "last"
+            },
+            {
+              "expression": "model",
+              "isExpression": false,
+              "asc": true,
+              "nulls": "last"
+            },
+            {
+              "expression": "dimensions",
+              "isExpression": false,
+              "asc": true,
+              "nulls": "last"
+            },
+            {
+              "expression": "metric",
+              "isExpression": false,
+              "asc": true,
+              "nulls": "last"
+            }
+          ],
+          "isUnique": false,
+          "concurrently": false,
+          "method": "btree",
+          "with": {}
+        }
+      },
+      "foreignKeys": {
+        "junior_memory_embeddings_memory_id_junior_memory_memories_id_fk": {
+          "name": "junior_memory_embeddings_memory_id_junior_memory_memories_id_fk",
+          "tableFrom": "junior_memory_embeddings",
+          "tableTo": "junior_memory_memories",
+          "columnsFrom": ["memory_id"],
+          "columnsTo": ["id"],
+          "onDelete": "cascade",
+          "onUpdate": "no action"
+        }
+      },
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {},
+      "policies": {},
+      "checkConstraints": {
+        "junior_memory_embeddings_metric_check": {
+          "name": "junior_memory_embeddings_metric_check",
+          "value": "\"junior_memory_embeddings\".\"metric\" IN ('cosine')"
+        },
+        "junior_memory_embeddings_dimensions_check": {
+          "name": "junior_memory_embeddings_dimensions_check",
+          "value": "\"junior_memory_embeddings\".\"dimensions\" = 1536"
+        }
+      },
+      "isRLSEnabled": false
+    },
+    "public.junior_memory_memories": {
+      "name": "junior_memory_memories",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "text",
+          "primaryKey": true,
+          "notNull": true
+        },
+        "scope": {
+          "name": "scope",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "scope_key": {
+          "name": "scope_key",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "type": {
+          "name": "type",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "subject_type": {
+          "name": "subject_type",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "subject_key": {
+          "name": "subject_key",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "content": {
+          "name": "content",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "search_vector": {
+          "name": "search_vector",
+          "type": "tsvector",
+          "primaryKey": false,
+          "notNull": false,
+          "generated": {
+            "as": "to_tsvector('english', \"content\")",
+            "type": "stored"
+          }
+        },
+        "source_platform": {
+          "name": "source_platform",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "source_key": {
+          "name": "source_key",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "idempotency_key": {
+          "name": "idempotency_key",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "observed_at_ms": {
+          "name": "observed_at_ms",
+          "type": "bigint",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "created_at_ms": {
+          "name": "created_at_ms",
+          "type": "bigint",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "expires_at_ms": {
+          "name": "expires_at_ms",
+          "type": "bigint",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "superseded_at_ms": {
+          "name": "superseded_at_ms",
+          "type": "bigint",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "superseded_by_id": {
+          "name": "superseded_by_id",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "archived_at_ms": {
+          "name": "archived_at_ms",
+          "type": "bigint",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "archive_reason": {
+          "name": "archive_reason",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        }
+      },
+      "indexes": {
+        "junior_memory_memories_visible_idx": {
+          "name": "junior_memory_memories_visible_idx",
+          "columns": [
+            {
+              "expression": "scope",
+              "isExpression": false,
+              "asc": true,
+              "nulls": "last"
+            },
+            {
+              "expression": "scope_key",
+              "isExpression": false,
+              "asc": true,
+              "nulls": "last"
+            },
+            {
+              "expression": "created_at_ms",
+              "isExpression": false,
+              "asc": false,
+              "nulls": "last"
+            },
+            {
+              "expression": "id",
+              "isExpression": false,
+              "asc": true,
+              "nulls": "last"
+            }
+          ],
+          "isUnique": false,
+          "where": "\"junior_memory_memories\".\"archived_at_ms\" IS NULL AND \"junior_memory_memories\".\"superseded_at_ms\" IS NULL AND \"junior_memory_memories\".\"superseded_by_id\" IS NULL",
+          "concurrently": false,
+          "method": "btree",
+          "with": {}
+        },
+        "junior_memory_memories_expiration_idx": {
+          "name": "junior_memory_memories_expiration_idx",
+          "columns": [
+            {
+              "expression": "expires_at_ms",
+              "isExpression": false,
+              "asc": true,
+              "nulls": "last"
+            }
+          ],
+          "isUnique": false,
+          "where": "\"junior_memory_memories\".\"archived_at_ms\" IS NULL AND \"junior_memory_memories\".\"expires_at_ms\" IS NOT NULL",
+          "concurrently": false,
+          "method": "btree",
+          "with": {}
+        },
+        "junior_memory_memories_search_idx": {
+          "name": "junior_memory_memories_search_idx",
+          "columns": [
+            {
+              "expression": "scope",
+              "isExpression": false,
+              "asc": true,
+              "nulls": "last"
+            },
+            {
+              "expression": "scope_key",
+              "isExpression": false,
+              "asc": true,
+              "nulls": "last"
+            },
+            {
+              "expression": "search_vector",
+              "isExpression": false,
+              "asc": true,
+              "nulls": "last"
+            }
+          ],
+          "isUnique": false,
+          "where": "\"junior_memory_memories\".\"archived_at_ms\" IS NULL AND \"junior_memory_memories\".\"superseded_at_ms\" IS NULL AND \"junior_memory_memories\".\"superseded_by_id\" IS NULL",
+          "concurrently": false,
+          "method": "gin",
+          "with": {}
+        },
+        "junior_memory_memories_idempotency_idx": {
+          "name": "junior_memory_memories_idempotency_idx",
+          "columns": [
+            {
+              "expression": "scope",
+              "isExpression": false,
+              "asc": true,
+              "nulls": "last"
+            },
+            {
+              "expression": "scope_key",
+              "isExpression": false,
+              "asc": true,
+              "nulls": "last"
+            },
+            {
+              "expression": "idempotency_key",
+              "isExpression": false,
+              "asc": true,
+              "nulls": "last"
+            }
+          ],
+          "isUnique": true,
+          "where": "\"junior_memory_memories\".\"idempotency_key\" IS NOT NULL AND \"junior_memory_memories\".\"archived_at_ms\" IS NULL AND \"junior_memory_memories\".\"superseded_at_ms\" IS NULL AND \"junior_memory_memories\".\"superseded_by_id\" IS NULL",
+          "concurrently": false,
+          "method": "btree",
+          "with": {}
+        }
+      },
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {},
+      "policies": {},
+      "checkConstraints": {
+        "junior_memory_memories_scope_check": {
+          "name": "junior_memory_memories_scope_check",
+          "value": "\"junior_memory_memories\".\"scope\" IN ('personal', 'conversation')"
+        },
+        "junior_memory_memories_kind_check": {
+          "name": "junior_memory_memories_kind_check",
+          "value": "\"junior_memory_memories\".\"type\" IN (\n        'preference',\n        'procedure',\n        'knowledge'\n      )"
+        },
+        "junior_memory_memories_subject_type_check": {
+          "name": "junior_memory_memories_subject_type_check",
+          "value": "\"junior_memory_memories\".\"subject_type\" IN ('user', 'conversation', 'general')"
+        },
+        "junior_memory_memories_subject_key_check": {
+          "name": "junior_memory_memories_subject_key_check",
+          "value": "(\"junior_memory_memories\".\"subject_type\" = 'general' AND \"junior_memory_memories\".\"subject_key\" IS NULL) OR (\"junior_memory_memories\".\"subject_type\" IN ('user', 'conversation') AND \"junior_memory_memories\".\"subject_key\" IS NOT NULL AND length(\"junior_memory_memories\".\"subject_key\") > 0)"
+        },
+        "junior_memory_memories_source_platform_check": {
+          "name": "junior_memory_memories_source_platform_check",
+          "value": "\"junior_memory_memories\".\"source_platform\" IN ('slack', 'local')"
+        }
+      },
+      "isRLSEnabled": false
+    }
+  },
+  "enums": {},
+  "schemas": {},
+  "sequences": {},
+  "roles": {},
+  "policies": {},
+  "views": {},
+  "_meta": {
+    "columns": {},
+    "schemas": {},
+    "tables": {}
+  }
+}

package/migrations/meta/_journal.json

@@ -43,6 +43,13 @@
       "when": 1785293874649,
       "tag": "0005_short_jamie_braddock",
       "breakpoints": true
+    },
+    {
+      "idx": 6,
+      "version": "7",
+      "when": 1785878952301,
+      "tag": "0006_friendly_hydra",
+      "breakpoints": true
     }
   ]
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sentry/junior-memory",
-  "version": "0.131.0",
+  "version": "0.133.0",
   "private": false,
   "publishConfig": {
     "access": "public"
@@ -27,7 +27,7 @@
     "commander": "^14.0.3",
     "drizzle-orm": "^0.45.2",
     "zod": "^4.4.3",
-    "@sentry/junior-plugin-api": "0.131.0"
+    "@sentry/junior-plugin-api": "0.133.0"
   },
   "devDependencies": {
     "@types/node": "^25.9.1",

package/src/db/schema.ts

@@ -8,6 +8,7 @@ import { sql } from "drizzle-orm";
 import {
   bigint,
   check,
+  customType,
   index,
   integer,
   pgTable,
@@ -24,6 +25,12 @@ import {
   MEMORY_KINDS,
 } from "../types";
 
+const tsvector = customType<{ data: string }>({
+  dataType() {
+    return "tsvector";
+  },
+});
+
 export const juniorMemoryMemories = pgTable(
   "junior_memory_memories",
   {
@@ -34,6 +41,9 @@ export const juniorMemoryMemories = pgTable(
     subjectType: text("subject_type", { enum: MEMORY_SUBJECT_TYPES }).notNull(),
     subjectKey: text("subject_key"),
     content: text("content").notNull(),
+    searchVector: tsvector("search_vector").generatedAlwaysAs(
+      sql`to_tsvector('english', "content")`,
+    ),
     sourcePlatform: text("source_platform", {
       enum: MEMORY_SOURCE_PLATFORMS,
     }).notNull(),
@@ -59,7 +69,7 @@ export const juniorMemoryMemories = pgTable(
         sql`${table.archivedAtMs} IS NULL AND ${table.expiresAtMs} IS NOT NULL`,
       ),
     index("junior_memory_memories_search_idx")
-      .using("gin", sql`to_tsvector('english', ${table.content})`)
+      .using("gin", table.scope, table.scopeKey, table.searchVector)
       .where(
         sql`${table.archivedAtMs} IS NULL AND ${table.supersededAtMs} IS NULL AND ${table.supersededById} IS NULL`,
       ),

package/src/store.ts

@@ -52,6 +52,8 @@ const DEFAULT_EXPIRED_ARCHIVE_LIMIT = 100;
 const PREFERENCE_ADJUDICATION_CANDIDATE_LIMIT = 10;
 const PREFERENCE_ADJUDICATION_VECTOR_LIMIT = 5;
 const VECTOR_SEARCH_OVERFETCH = 4;
+const LEXICAL_RANK_OVERFETCH = 4;
+const MAX_LEXICAL_RANK_CANDIDATES = 1_000;
 const MAX_MEMORY_CONTENT_CHARS = 4_000;
 const EMBEDDING_METRIC = "cosine";
 
@@ -128,6 +130,7 @@ const memoryRowSchema = z
     id: z.string().min(1),
     idempotencyKey: optionalStringSchema,
     observedAtMs: z.coerce.number(),
+    searchVector: z.string().optional(),
     scope: z.enum(MEMORY_SCOPES),
     scopeKey: z.string().min(1),
     sourceKey: z.string().min(1),
@@ -946,7 +949,7 @@ async function listVisibleMemories(args: {
   return rows.map(parseMemoryRow);
 }
 
-/** Search active visible records with PostgreSQL full-text ranking. */
+/** Search a bounded active candidate set with PostgreSQL full-text ranking. */
 async function searchVisibleLexicalMemories(args: {
   db: MemoryDb;
   limit: number;
@@ -966,20 +969,49 @@ async function searchVisibleLexicalMemories(args: {
     )
     FROM unnest(tsvector_to_array(${queryVector})) AS query_terms(term)
   )`;
-  const textsearch = sql`to_tsvector('english', ${juniorMemoryMemories.content})`;
-  const textRank = sql<number>`ts_rank_cd(${textsearch}, ${tsquery})`;
-  const rows = await args.db
-    .select({
-      memory: juniorMemoryMemories,
-      textRank,
-    })
+  const candidateLimit = Math.min(
+    MAX_LEXICAL_RANK_CANDIDATES,
+    args.limit * LEXICAL_RANK_OVERFETCH,
+  );
+  const candidates = args.db
+    .select()
     .from(juniorMemoryMemories)
-    .where(and(predicate, sql`${textsearch} @@ ${tsquery}`))
+    .where(
+      and(predicate, sql`${juniorMemoryMemories.searchVector} @@ ${tsquery}`),
+    )
     .orderBy(
-      desc(textRank),
       desc(juniorMemoryMemories.observedAtMs),
       asc(juniorMemoryMemories.id),
     )
+    .limit(candidateLimit)
+    .as("lexical_candidates");
+  const textRank = sql<number>`ts_rank_cd(${candidates.searchVector}, ${tsquery})`;
+  const rows = await args.db
+    .select({
+      memory: {
+        archiveReason: candidates.archiveReason,
+        archivedAtMs: candidates.archivedAtMs,
+        content: candidates.content,
+        createdAtMs: candidates.createdAtMs,
+        expiresAtMs: candidates.expiresAtMs,
+        id: candidates.id,
+        idempotencyKey: candidates.idempotencyKey,
+        kind: candidates.kind,
+        observedAtMs: candidates.observedAtMs,
+        scope: candidates.scope,
+        scopeKey: candidates.scopeKey,
+        searchVector: candidates.searchVector,
+        sourceKey: candidates.sourceKey,
+        sourcePlatform: candidates.sourcePlatform,
+        subjectKey: candidates.subjectKey,
+        subjectType: candidates.subjectType,
+        supersededAtMs: candidates.supersededAtMs,
+        supersededById: candidates.supersededById,
+      },
+      textRank,
+    })
+    .from(candidates)
+    .orderBy(desc(textRank), desc(candidates.observedAtMs), asc(candidates.id))
     .limit(args.limit);
   const ranks = denseRanks(rows, (row) => Number(row.textRank));
   return rows.map((row, index) => ({