@martian-engineering/lossless-claw 0.7.0 → 0.8.1

package/src/store/fts5-sanitize.ts

@@ -1,50 +0,0 @@
-/**
- * Sanitize a user-provided query for use in an FTS5 MATCH expression.
- *
- * FTS5 treats certain characters as operators:
- *   - `-` (NOT), `+` (required), `*` (prefix), `^` (initial token)
- *   - `OR`, `AND`, `NOT` (boolean operators)
- *   - `:` (column filter — e.g. `agent:foo` means "search column agent")
- *   - `"` (phrase query), `(` `)` (grouping)
- *   - `NEAR` (proximity)
- *
- * If the query contains any of these, naive MATCH will either error
- * ("no such column") or return unexpected results.
- *
- * Strategy: wrap each whitespace-delimited token in double quotes so FTS5
- * treats it as a literal phrase token. Internal double quotes are stripped.
- * Empty tokens are dropped. Tokens are joined with spaces (implicit AND).
- *
- * Examples:
- *   "sub-agent restrict"  →  '"sub-agent" "restrict"'
- *   "lcm_expand OR crash" →  '"lcm_expand" "OR" "crash"'
- *   'hello "world"'       →  '"hello" "world"'
- */
-export function sanitizeFts5Query(raw: string): string {
-  // Preserve user-quoted phrases: extract "..." groups first, then tokenize the rest.
-  const parts: string[] = [];
-  const phraseRegex = /"([^"]+)"/g;
-  let match: RegExpExecArray | null;
-  let lastIndex = 0;
-
-  while ((match = phraseRegex.exec(raw)) !== null) {
-    // Process unquoted text before this phrase
-    const before = raw.slice(lastIndex, match.index);
-    for (const t of before.split(/\s+/).filter(Boolean)) {
-      parts.push(`"${t.replace(/"/g, "")}"`);
-    }
-    // Preserve the phrase as-is (strip internal quotes for safety)
-    const phrase = match[1].replace(/"/g, "").trim();
-    if (phrase) {
-      parts.push(`"${phrase}"`);
-    }
-    lastIndex = match.index + match[0].length;
-  }
-
-  // Process unquoted text after last phrase
-  for (const t of raw.slice(lastIndex).split(/\s+/).filter(Boolean)) {
-    parts.push(`"${t.replace(/"/g, "")}"`);
-  }
-
-  return parts.length > 0 ? parts.join(" ") : '""';
-}

package/src/store/full-text-fallback.ts

@@ -1,83 +0,0 @@
-const RAW_TERM_RE = /"([^"]+)"|(\S+)/g;
-
-/**
- * Detect whether a query contains CJK characters that FTS5 unicode61
- * tokenizer cannot index properly.
- */
-const CJK_RE = /[\u2E80-\u9FFF\u3400-\u4DBF\uF900-\uFAFF\uAC00-\uD7AF\u3040-\u309F\u30A0-\u30FF]/;
-export function containsCjk(text: string): boolean {
-  return CJK_RE.test(text);
-}
-const EDGE_PUNCTUATION_RE = /^[`'"()[\]{}<>.,:;!?*_+=|\\/-]+|[`'"()[\]{}<>.,:;!?*_+=|\\/-]+$/g;
-
-export type LikeSearchPlan = {
-  terms: string[];
-  where: string[];
-  args: string[];
-};
-
-function normalizeFallbackTerm(raw: string): string {
-  return raw.trim().replace(EDGE_PUNCTUATION_RE, "").toLowerCase();
-}
-
-function escapeLike(term: string): string {
-  return term.replace(/([\\%_])/g, "\\$1");
-}
-
-/**
- * Convert a free-text query into a conservative LIKE search plan.
- *
- * The fallback keeps phrase tokens when the query uses double quotes, and
- * otherwise searches for all normalized tokens as case-insensitive substrings.
- */
-export function buildLikeSearchPlan(column: string, query: string): LikeSearchPlan {
-  const terms: string[] = [];
-  for (const match of query.matchAll(RAW_TERM_RE)) {
-    const raw = match[1] ?? match[2] ?? "";
-    const normalized = normalizeFallbackTerm(raw);
-    if (normalized.length > 0 && !terms.includes(normalized)) {
-      terms.push(normalized);
-    }
-  }
-
-  if (terms.length === 0) {
-    const fallback = normalizeFallbackTerm(query);
-    if (fallback.length > 0) {
-      terms.push(fallback);
-    }
-  }
-
-  return {
-    terms,
-    where: terms.map(() => `LOWER(${column}) LIKE ? ESCAPE '\\'`),
-    args: terms.map((term) => `%${escapeLike(term)}%`),
-  };
-}
-
-/**
- * Build a compact snippet centered around the earliest matching term.
- */
-export function createFallbackSnippet(content: string, terms: string[]): string {
-  const haystack = content.toLowerCase();
-  let matchIndex = -1;
-  let matchLength = 0;
-
-  for (const term of terms) {
-    const idx = haystack.indexOf(term);
-    if (idx !== -1 && (matchIndex === -1 || idx < matchIndex)) {
-      matchIndex = idx;
-      matchLength = term.length;
-    }
-  }
-
-  if (matchIndex === -1) {
-    const head = content.trim();
-    return head.length <= 80 ? head : `${head.slice(0, 77).trimEnd()}...`;
-  }
-
-  const start = Math.max(0, matchIndex - 24);
-  const end = Math.min(content.length, matchIndex + Math.max(matchLength, 1) + 40);
-  const prefix = start > 0 ? "..." : "";
-  const suffix = end < content.length ? "..." : "";
-  return `${prefix}${content.slice(start, end).trim()}${suffix}`;
-}

package/src/store/full-text-sort.ts

@@ -1,21 +0,0 @@
-export type SearchSort = "recency" | "relevance" | "hybrid";
-
-export const AGE_DECAY_RATE = 0.001;
-
-/**
- * Build the ORDER BY clause for FTS5-backed searches.
- *
- * `rank` is FTS5's BM25 score where lower (more negative) is better.
- * `hybrid` keeps that relevance signal but applies a mild age penalty before
- * LIMIT is enforced so older strong matches can still surface.
- */
-export function buildFtsOrderBy(sort: SearchSort | undefined, createdAtExpr: string): string {
-  switch (sort ?? "recency") {
-    case "relevance":
-      return `rank ASC, ${createdAtExpr} DESC`;
-    case "hybrid":
-      return `(rank / (1 + ((julianday('now') - julianday(${createdAtExpr})) * 24 * ${AGE_DECAY_RATE}))) ASC, ${createdAtExpr} DESC`;
-    default:
-      return `${createdAtExpr} DESC`;
-  }
-}

package/src/store/index.ts

@@ -1,39 +0,0 @@
-export { ConversationStore } from "./conversation-store.js";
-export type {
-  ConversationId,
-  MessageId,
-  SummaryId,
-  MessageRole,
-  MessagePartType,
-  MessageRecord,
-  MessagePartRecord,
-  ConversationRecord,
-  CreateMessageInput,
-  CreateMessagePartInput,
-  CreateConversationInput,
-  MessageSearchInput,
-  MessageSearchResult,
-} from "./conversation-store.js";
-
-export { SummaryStore } from "./summary-store.js";
-export type {
-  SummaryKind,
-  ContextItemType,
-  CreateSummaryInput,
-  SummaryRecord,
-  ContextItemRecord,
-  SummarySearchInput,
-  SummarySearchResult,
-  CreateLargeFileInput,
-  LargeFileRecord,
-  UpsertConversationBootstrapStateInput,
-  ConversationBootstrapStateRecord,
-} from "./summary-store.js";
-
-export { CompactionTelemetryStore } from "./compaction-telemetry-store.js";
-export type {
-  CacheState,
-  ActivityBand,
-  ConversationCompactionTelemetryRecord,
-  UpsertConversationCompactionTelemetryInput,
-} from "./compaction-telemetry-store.js";

package/src/store/parse-utc-timestamp.ts

@@ -1,25 +0,0 @@
-/**
- * Parse a SQLite UTC timestamp string into a Date object.
- * SQLite stores timestamps via datetime('now') without a Z suffix,
- * which causes JS to parse them as local time instead of UTC.
- * See: https://github.com/Martian-Engineering/lossless-claw/issues/216
- */
-export function parseUtcTimestamp(value: string): Date {
-  const s = value.trim();
-  if (/(?:[zZ]|[+-]\d{2}:\d{2})$/.test(s)) {
-    return new Date(s);
-  }
-
-  const normalized = s.includes("T") ? s : s.replace(" ", "T");
-  return new Date(`${normalized}Z`);
-}
-
-/**
- * Parse a nullable SQLite UTC timestamp string into a Date object.
- */
-export function parseUtcTimestampOrNull(
-  value: string | null | undefined,
-): Date | null {
-  if (value == null) return null;
-  return parseUtcTimestamp(value);
-}