lossless-core 0.1.2 → 0.1.3

package/CHANGELOG.md

@@ -1,5 +1,16 @@
 # lossless-core
 
+## 0.1.3
+
+### Patch Changes
+
+- 470a7b7: Verify file bytes and sha256 against the linked message text before inserting file rows on import. A repeated entry id with different text, or a file row whose text differs from the stored message, also refuses the import without writing. Existing rows stay untouched.
+- 470a7b7: Signal a watermark miss on the ingest path. A full rescan now writes one `ingest-rescan` metric row and, in interactive sessions, one UI warning. Empty entries stay silent and ingest behavior is unchanged.
+- 470a7b7: Add a mechanical tier for noisy tools. Name tools in `mechanicalTools` in summarizer-config, `/lcm` config, or the OpenCode plugin options. Their spans compact into deterministic `mechanical` leaves with zero summarizer spend at compaction time. Every message stays stored verbatim and covered, so the integrity coverage check passes, and heal re-derives a skipped leaf from provenance on demand. There is no ingest filtering.
+- 470a7b7: Report orphan sidecars in doctor. `integritySnapshot` now lists sidecar files and stale `wal` and `shm` files with no live session (`.sqlite` by default, `.db` for Pi). `checkIntegrity` reports them as a read-only `orphan-sidecar` finding through `lcm doctor` and the diagnose bundle. Report only. Deletion stays manual.
+- 470a7b7: Upgrade recall search. `lcm_grep` takes `role`, `date_start`/`date_end`, and `tool_name` filters. Role and time are SQL predicates on the stored columns with a covering index. Tool scope is an FTS rewrite against `[tool:Name]` markers with no new table or index. `lcm_expand` takes an `anchor` entry id that centers the listing window with edge clamping. The recall manifest and the retrieval brief order by stop-word stripped query-word overlap over summary headers. A read-only files view lists each linked file once with an incomplete-coverage flag and a `large-inline` mark. No migration and no new writer.
+- 0c70748: Ask providers for a whole number of tokens. A leaf target was half of the calibrated chunk budget, so it was fractional whenever that budget was odd, and a provider that validates `max_tokens` as an unsigned integer rejected the call with a 422. The ladder then stored the deterministic truncate: in one week of Pi sessions that was 1,315 of 5,358 summaries, with 540 rejected calls naming values such as `max_tokens: 5543.5`. The request builder now rounds to nearest, never below one, and treats a non-finite budget as that floor, before the reasoning headroom is added. Integer budgets produce the same request as before. No config key, migration or metric row changes.
+
 ## 0.1.2
 
 ### Patch Changes

package/dist/index.d.mts

@@ -198,6 +198,15 @@ export interface IntegritySnapshot {
   foreignKeyViolations: number;
   schemaVersion: number;
   uncoveredMessages: number;
+  /** Full paths, sorted, so doctor can name what a manual drop would remove. */
+  orphanSidecars: string[];
+}
+export interface IntegritySnapshotOptions {
+  /** Paths with an open store in this process; every other sidecar file in
+   * `sidecarDir` reports as unopened, even when its session still exists. */
+  liveSidecarPaths?: readonly string[];
+  sidecarDir?: string;
+  sidecarExtension?: string;
 }
 /** A fresh row, or the row that already owns the span; `existingTokens` comes
  * from that row, so a caller reports the collision, not its intended write.
@@ -454,14 +463,23 @@ export declare class LcmStore {
     offset?: number;
     withinMessageIds?: ReadonlySet<number>;
     includeRemoved?: boolean;
+    role?: LcmRole;
+    dateStart?: number;
+    dateEnd?: number;
   }): GrepHit[];
   grepCount(pattern: string, opts?: {
     withinMessageIds?: ReadonlySet<number>;
     includeRemoved?: boolean;
+    role?: LcmRole;
+    dateStart?: number;
+    dateEnd?: number;
   }): number | undefined;
   messagePage(afterRowId: number, limit: number, opts?: {
     withinMessageIds?: ReadonlySet<number>;
     includeRemoved?: boolean;
+    role?: LcmRole;
+    dateStart?: number;
+    dateEnd?: number;
   }): ScannedRow[];
   coveringSummaryId(entryId: string): number | undefined;
   messageByEntryId(entryId: string): StoredMessage | undefined;
@@ -478,7 +496,7 @@ export declare class LcmStore {
   getSummary(id: number, run?: string): SummaryNode | undefined;
   /** Bounded by a visited set, so a corrupt cycle cannot recurse forever. */
   coveredMessageIds(summaryId: number, seen?: Set<number>): number[];
-  integritySnapshot(): IntegritySnapshot;
+  integritySnapshot(opts?: IntegritySnapshotOptions): IntegritySnapshot;
   /** Restore the full-text index. The schema is re-applied first, because a
    * dropped virtual table cannot be rebuilt, only recreated, and the DDL that
    * knows how is the schema itself. Everything in it is `IF NOT EXISTS` apart
@@ -538,10 +556,12 @@ export declare const NO_STORE = "LCM: no stored history for this session yet (no
 export declare const RECALL_SCOPES: readonly ["session", "sessions", "all_sessions"];
 /** One of `RECALL_SCOPES`; `sessions` and `all_sessions` are equivalent and both read past sidecars. */
 export type RecallScope = (typeof RECALL_SCOPES)[number];
-/** What `lcm_expand` reads: a summary listing by id, or one message by entry id with an optional summary check. */
+/** What `lcm_expand` reads: a summary listing by id (optionally centered on an
+ * anchor entry), or one message by entry id with an optional summary check. */
 export type Address = {
   readonly kind: "summary";
   readonly id: number;
+  readonly anchor?: string;
 } | {
   readonly kind: "entry";
   readonly entryId: string;
@@ -569,18 +589,27 @@ export interface RecallToolDeps {
   readonly record?: (row: Record<string, unknown>) => void;
   readonly sessionOf?: (sessionId: string) => string;
   readonly isReader?: (sessionId: string) => boolean;
-  readonly searchPast?: (session: string, query: string, scope: Exclude<RecallScope, "session">, limit: unknown) => RecallPastSearchResult;
+  readonly searchPast?: (session: string, query: string, scope: Exclude<RecallScope, "session">, limit: unknown, filters?: GrepFilterArgs) => RecallPastSearchResult;
   readonly openPast?: (session: string, pastSession: string) => OpenPastResult;
   readonly pastSearchUnreachableText?: string;
   readonly pastReadUnreachableText?: string;
 }
 /** Answer `lcm_grep`: the session `grepView`, or past sidecars when the scope names them; readers stay session-scoped. */
 export declare function grepAnswer(deps: RecallToolDeps, callerSessionID: string, input: unknown): string;
+/** Optional `lcm_grep` filters, shared by the session search and the past-session
+ * scan. Unknown roles and bad timestamps are caller errors; an empty tool name
+ * is too, while an unknown-but-well-formed tool name simply matches nothing. */
+export interface GrepFilterArgs {
+  role?: "user" | "assistant" | "toolResult" | "custom";
+  dateStart?: number;
+  dateEnd?: number;
+  toolName?: string;
+}
 /** Answer `lcm_describe`: the redacted cost and contents of one summary id, or why it missed. */
 export declare function describeAnswer(deps: RecallToolDeps, callerSessionID: string, input: unknown): string;
 /** Answer `lcm_expand`: list a summary or read one message, capped at `MAX_EXPAND_CHARS`; a foreign `session` id reads a past sidecar read-only. */
 export declare function expandAnswer(deps: RecallToolDeps, callerSessionID: string, input: unknown): string;
-/** Parse `id`/`entry_id` args into an `Address`: error text when malformed, undefined when neither is present. */
+/** Parse `id`/`entry_id`/`anchor` args into an `Address`: error text when malformed, undefined when neither is present. */
 export declare function readAddress(args: Record<string, unknown>): Address | string | undefined;
 /** Refusal text when an address names no stored summary or message, else undefined. */
 export declare function addressProblem(store: LcmStore, address: Address): string | undefined;
@@ -1239,6 +1268,12 @@ export type BranchRemovedRow = LcmRowBase & {
   removed: number;
   restored: number;
 };
+/** A watermark miss forced a full rescan. Empty deltas never write one. */
+export type IngestRescanRow = LcmRowBase & {
+  kind: "ingest-rescan";
+  watermark: string;
+  entries: number;
+};
 export type FileExternalizedRow = LcmRowBase & {
   kind: "file-externalized";
   entryId: string;
@@ -1310,7 +1345,7 @@ export type ImportRow = MetricBase & {
   summariesInserted: number;
   summariesSkipped: number;
 });
-export type LcmMetricRecord = ContextDecisionRow | ProjectionOverBudgetRow | SwapAppliedRow | SwapBlockedRow | PiCompactionRow | EstimateCalibratedRow | WindowCorrectedRow | WindowFloorRow | ConfigProblemRow | SummaryInputClippedRow | SummaryFallbackStoredRow | SummaryStoredRow | SummarySkippedCoveredRow | SummarySkippedExistingRow | SummaryOversizedRow | SummaryReplacedRow | SummaryFinalTruncatedRow | CondensationStalledRow | CondensationReducedRow | CompactionCompleteRow | HealStartRow | HealNodeRow | HealCompleteRow | SummarizerErrorRow | SummarizerCappedRow | SummarizerFallbackRow | SummarizerModelRow | SummarizerUsageRow | CompactionErrorRow | CompactionAwaitedRow | CompactionQueuedRow | CompactionQueuedDroppedRow | CompactionTimeoutClearedRow | CompactionStaleRow | CompactionAbandonedRow | CompactionNoopRow | GcDeletedRow | StoreMigratedRow | BranchRemovedRow | FileExternalizedRow | LargeInlineRow | RecallRow | RetrievalRow | EvalRow | ImportRow;
+export type LcmMetricRecord = ContextDecisionRow | ProjectionOverBudgetRow | SwapAppliedRow | SwapBlockedRow | PiCompactionRow | EstimateCalibratedRow | WindowCorrectedRow | WindowFloorRow | ConfigProblemRow | SummaryInputClippedRow | SummaryFallbackStoredRow | SummaryStoredRow | SummarySkippedCoveredRow | SummarySkippedExistingRow | SummaryOversizedRow | SummaryReplacedRow | SummaryFinalTruncatedRow | CondensationStalledRow | CondensationReducedRow | CompactionCompleteRow | HealStartRow | HealNodeRow | HealCompleteRow | SummarizerErrorRow | SummarizerCappedRow | SummarizerFallbackRow | SummarizerModelRow | SummarizerUsageRow | CompactionErrorRow | CompactionAwaitedRow | CompactionQueuedRow | CompactionQueuedDroppedRow | CompactionTimeoutClearedRow | CompactionStaleRow | CompactionAbandonedRow | CompactionNoopRow | GcDeletedRow | StoreMigratedRow | BranchRemovedRow | IngestRescanRow | FileExternalizedRow | LargeInlineRow | RecallRow | RetrievalRow | EvalRow | ImportRow;
 export type MetricRecord = UsageRow | LcmMetricRecord;
 export interface LlmComplete {
   (systemPrompt: string, userText: string, maxTokens: number): Promise<string>;
@@ -1336,6 +1371,8 @@ export declare function summarizeTerse(items: Array<{
   role: string;
   text: string;
 }>, targetTokens: number, llm: LlmComplete, session?: string): Promise<EscalationResult>;
+/** Same budget the ladder falls back to, so heal sees one shape. */
+export declare const LEVEL3_TOKENS = 512;
 export declare function deterministicTruncate(text: string, targetTokens: number): string;
 export interface SummarizerConfig {
   /** One model id, or an ordered fallback chain. `auto` is the session model. */
@@ -1439,7 +1476,32 @@ export interface SummarizerOptions {
   readonly summarizerTokensPerPass?: number;
   /** Tokens one on-demand heal run may spend across all of its calls. */
   readonly summarizerTokensPerHeal?: number;
-}
+  /** Fires at compaction time only; ingest still stores every message verbatim. */
+  readonly mechanicalTools?: readonly string[];
+}
+/** Empty default: callers opt in and existing passes keep their spend. */
+export declare const DEFAULT_MECHANICAL_TOOLS: readonly string[];
+export declare function normalizeMechanicalTools(tools: readonly string[] | undefined): string[];
+/** The tool name a message carries, if any. Reads both hosts: `tool <name>`
+ * (OpenCode), `[tool:<name>]` (core), `tool-call: <name>`, plus a JSON
+ * payload that names the call. Lowercased, else undefined.
+ *
+ * A marker only counts at a block start: `entryToText` joins content blocks
+ * with newlines, so a genuine call marker opens a line while prose quoting one
+ * sits mid-line. The caller may pass the stored role: `user` and `custom` rows
+ * never carry calls, and an `assistant` row mixing prose with a call keeps the
+ * prose on the summarizer path (tool-only rows still skip). */
+export declare function mechanicalToolNameOf(message: {
+  text: string;
+  payload?: string;
+  role?: string;
+}): string | undefined;
+/** Normalized here, so callers may pass raw config values. */
+export declare function isMechanicalToolMessage(message: {
+  text: string;
+  payload?: string;
+  role?: string;
+}, tools: readonly string[]): boolean;
 /** The engine config for these options. An unresolved budget keeps the core
  * default, which is also what a direct caller that passes no options gets. */
 export declare function summarizerConfig(options: SummarizerOptions): SummarizerConfig;
@@ -1674,6 +1736,8 @@ export interface CompactionInput {
   targetTokens?: number;
   /** Where the returned summary must reach back to; defaults to the span start. */
   frontierFrom?: string;
+  /** Read at compaction time only; ingest stores every message verbatim. */
+  mechanicalTools?: readonly string[];
 }
 export type CompactionOutcome = {
   kind: "stored";
@@ -1681,6 +1745,10 @@ export type CompactionOutcome = {
   leafSummaryIds: number[];
   condensedSummaryIds: number[];
   ingested: number;
+  /** Leaves stored without a model call on the configured mechanical path. */
+  mechanicalLeaves?: number;
+  /** Condensed nodes built without a model call from mechanical children. */
+  mechanicalCondensed?: number;
 } |
 /** No text, and why. A span another live run claimed, a span an existing node
  * already covers, and a span the store holds no row for are all states a pass
@@ -1705,6 +1773,8 @@ export declare function runCompaction(store: PassStore, input: CompactionInput,
   session?: string;
   maxDepth?: number;
   stage?: CompactionStage;
+  /** Empty keeps the LLM path. */
+  mechanicalTools?: readonly string[];
 }): Promise<CompactionOutcome>;
 /** The next group to condense: the oldest run of adjacent same-depth nodes, at
  * the shallowest depth with a run of two or more, capped at MAX_DAG_FANOUT.
@@ -1820,7 +1890,7 @@ export interface CompactionBounds {
  * not message text, so bash and branch-summary rows align. An unknown
  * firstKeptEntryId yields an empty span, so the caller falls back to Pi. */
 export declare function resolveCompactionSpan(branchMsgs: readonly AlignedEntry[], bounds: CompactionBounds): AlignedEntry[];
-export type FindingKind = "fts-index" | "summary-span" | "summary-shape" | "orphan-child" | "duplicate-span" | "foreign-key" | "schema-version" | "dead-run" | "coverage";
+export type FindingKind = "fts-index" | "summary-span" | "summary-shape" | "orphan-child" | "duplicate-span" | "foreign-key" | "schema-version" | "dead-run" | "orphan-sidecar" | "coverage";
 export interface Finding {
   kind: FindingKind;
   /** `violation` is an invariant the DAG depends on; `coverage` is a count. */
@@ -1834,7 +1904,7 @@ export interface IntegrityReport {
   checks: number;
   findings: Finding[];
 }
-export declare function checkIntegrity(store: LcmStore): IntegrityReport;
+export declare function checkIntegrity(store: LcmStore, opts?: IntegritySnapshotOptions): IntegrityReport;
 /** Fix what can be fixed without guessing: the derived index, rows whose span no
  * longer exists, nodes whose child no longer resolves, rows a run whose writer is
  * gone left behind, and predated duplicate spans. An inverted span is left
@@ -2328,6 +2398,40 @@ export type ViewResult = {
   ok: false;
   text: string;
 };
+declare const LCM_ROLES: readonly ["user", "assistant", "toolResult", "custom"];
+type GrepFilter = {
+  role?: (typeof LCM_ROLES)[number];
+  dateStart?: number;
+  dateEnd?: number;
+  toolName?: string;
+};
+/** The `[tool:Name]` marker in front of a tool call's arguments, so a scoped
+ * search needs no table or index. */
+export declare function toolMarker(toolName: string): string;
+/** Quote a string as one FTS5 phrase. Embedded quotes double; JSON.stringify
+ * emits backslash escapes which FTS5 rejects. */
+export declare function ftsPhrase(text: string): string;
+/** Rewrite an FTS5 query to its tool-scoped form. The marker passes as a quoted
+ * phrase, which the FTS5 tokenizer splits the same way as the stored text, so
+ * prose with the same adjacent words (for example "which tool read the file")
+ * can also match. Anchoring to the row start would miss mixed blocks because
+ * entryToText appends blocks in order, so the scoped search keeps this broader
+ * match and callers read the hit to confirm the tool. */
+export declare function withToolScope(query: string, toolName: string): string;
+/** Row ids a tool scope covers: the call rows that name the tool plus the
+ * toolResult rows that follow them. Pi stores a call and its result in
+ * separate rows, so a same-row `query AND marker` FTS misses a word found
+ * only in the result. A trailing run of toolResult rows after a marker row
+ * belongs to that call batch; any other role ends the run, as does a
+ * toolResult row that names a different tool. Bracket text is checked
+ * directly so one row naming two tools anchors both, and
+ * mechanicalToolNameOf covers the `tool <name>` and payload forms. User and
+ * custom rows never open a scope, so quoting a marker in chat pulls in no
+ * results; a mixed assistant block still opens through its literal marker. */
+export declare function toolScopeMessageIds(store: LcmStore, toolName: string, opts?: {
+  includeRemoved?: boolean;
+  withinMessageIds?: ReadonlySet<number>;
+}): Set<number>;
 export declare function grepView(store: LcmStore, redact: (text: string) => string, params: {
   query?: string;
   pattern?: string;
@@ -2342,17 +2446,37 @@ export declare function grepView(store: LcmStore, redact: (text: string) => stri
   now?: () => number;
   /** Tool the continuation marker names. Defaults to `lcm_expand_query`. */
   expandTool?: string;
+  role?: GrepFilter["role"];
+  dateStart?: number;
+  dateEnd?: number;
+  /** Query searches only; with a pattern it is refused. */
+  toolName?: string;
 }): ViewResult;
+/** A scoped FTS search that stays complete. `store.grep` applies its SQL
+ * LIMIT before it intersects the id set, so one bounded call can miss rows
+ * that rank past the window. The unscoped count bounds the rank order, so one
+ * fetch of that size holds every in-scope match and the caller pages the
+ * filtered set. Rank order is preserved. Throws on invalid FTS5 syntax, like
+ * `store.grep`, so callers keep their quoted-phrase fallback. */
+export declare function scopedGrepAll(store: LcmStore, pattern: string, opts: {
+  includeRemoved: boolean;
+  role?: GrepFilter["role"];
+  dateStart?: number;
+  dateEnd?: number;
+  scoped: ReadonlySet<number>;
+}): ReturnType<LcmStore["grep"]>;
 export declare function describeView(store: LcmStore, redact: (text: string) => string, id: number): ViewResult;
-/** List the summary's messages, or read one of them as a window of characters.
- * A message read without a summary names an entry directly, which is how an
- * entry that no leaf covers yet is read at all. */
+/** List a summary's messages, or read one as a window. Without a summary an
+ * entry id reads directly; with `anchor` and offset 0 the window centers on
+ * that entry, clamped at the span edges. A nonzero offset with `anchor` is an
+ * absolute message index, so the next page starts from the reported offset. */
 export type ExpandParams = {
   mode: "list";
   id: number;
   offset: number;
   limit: number;
   maxChars: number;
+  anchor?: string;
 } | {
   mode: "message";
   id?: number;
@@ -2361,7 +2485,20 @@ export type ExpandParams = {
   maxChars: number;
 };
 export declare function expandView(store: LcmStore, redact: (text: string) => string, params: ExpandParams): ViewResult;
-export declare function historyManifest(store: LcmStore, limit: number): string;
+/** Sort headers the caller already holds, so FTS plus regex stay the recall
+ * path and no score is stored. Ties keep input order. */
+export declare function orderSummariesByOverlap<T extends {
+  text: string;
+}>(query: string, nodes: readonly T[]): T[];
+/** At or above this length with no linked file a body reads as `large-inline`,
+ * matching the ingest default, so view and writer agree. */
+export declare const LARGE_INLINE_CHARS = 32000;
+/** A read-only files manifest, built at read time, so nothing is stored and
+ * no model is called. */
+export declare function filesView(store: LcmStore, redact: (text: string) => string, opts?: {
+  summaryId?: number;
+}): ViewResult;
+export declare function historyManifest(store: LcmStore, limit: number, query?: string): string;
 /** Redactor that re-reads the flag on every call, so a cockpit toggle takes
  * effect immediately: a getter, because patchConfig replaces the config object
  * and closing over it would pin the session-start value. */
@@ -2508,6 +2645,10 @@ export declare function searchSessions(refs: readonly SessionRef[], query: strin
   deadlineMs?: number;
   perSession?: number;
   now?: () => number;
+  role?: "user" | "assistant" | "toolResult" | "custom";
+  dateStart?: number;
+  dateEnd?: number;
+  toolName?: string;
 }): SessionSearchResult;
 export declare const SEARCH_SCOPE_HELP = "session (default) searches this session. sessions searches past sessions in this project's working directory. all_sessions searches past sessions in every project. A cross-session hit is a pointer labelled [lcm:session <hash>] that lcm_expand_query({session, entry_id}) reads, so a past session's text is never injected into the context on its own.";
 /** The cross-session hits as one labelled block, plus what the scan did not read.