nebulosity 1.0.0 → 1.1.0

package/lib/db/migrations/meta/_journal.json

@@ -78,6 +78,13 @@
       "when": 1792890069157,
       "tag": "0010_e4_declares",
       "breakpoints": true
+    },
+    {
+      "idx": 11,
+      "version": "6",
+      "when": 1793261385735,
+      "tag": "0011_plain_summaries_prompt_version",
+      "breakpoints": true
     }
   ]
 }

package/lib/db/schema.ts

@@ -156,9 +156,10 @@ export const summaries = sqliteTable("summaries", {
     targetKind: text("target_kind", { enum: SUMMARY_TARGET_KINDS }).notNull(),
     targetPath: text("target_path").notNull(),   // "" for the repo itself
     tagline: text("tagline"),                     // 3–6 words asked for, up to 8 kept whole; null until a summary exists
-    summary: text("summary"),                     // 2–4 sentences
+    summary: text("summary"),                     // short paragraphs, separated by blank lines
     model: text("model"),                         // the model that wrote it, or "template"
     inputHash: text("input_hash"),                // hash(model, prompt version, input) of the text above
+    promptVersion: integer("prompt_version"),     // the prompt version that wrote the text above, null for a template or for text from before v3 (ADR-034)
     status: text("status", { enum: SUMMARY_STATUSES }).notNull(),
     error: text("error"),                         // why the last attempt failed
     // A Message Batch can take hours. These two say which batch carries this row's request and

package/lib/db/store/map.ts

@@ -16,6 +16,10 @@ export type SummaryState = {
     path: string;
     tagline: string | null;
     summary: string | null;
+    /** The model that wrote the text, or "template". */
+    model: string | null;
+    /** The prompt version that wrote the text, null for a template or for text from before v3 (ADR-034). */
+    promptVersion: number | null;
     status: StoredSummaryStatus;
     error: string | null;
 };
@@ -40,6 +44,8 @@ const summaryColumns = {
     path: summaries.targetPath,
     tagline: summaries.tagline,
     summary: summaries.summary,
+    model: summaries.model,
+    promptVersion: summaries.promptVersion,
     status: summaries.status,
     error: summaries.error,
 };

package/lib/db/store/summaries.ts

@@ -20,6 +20,8 @@ export type StoredSummary = {
     summary: string | null;
     model: string | null;
     inputHash: string | null;
+    /** The prompt version that wrote the text, null for a template or for text from before v3 (ADR-034). */
+    promptVersion: number | null;
     status: StoredSummaryStatus;
     batchId: string | null;
     error: string | null;
@@ -53,7 +55,7 @@ export type SummaryResult = {
     batchId: string | null;
     /** The input the answer was written for. */
     inputHash: string;
-    outcome: { ok: true; text: SummaryText; model: string } | { ok: false; error: string };
+    outcome: { ok: true; text: SummaryText; model: string; promptVersion: number } | { ok: false; error: string };
 };
 
 export type UsageRow = { model: string; transport: "direct" | "batch"; requests: number; inputTokens: number; outputTokens: number };
@@ -117,6 +119,7 @@ export function loadSummaries(db: Db, repoId: number): StoredSummary[] {
         summary: summaries.summary,
         model: summaries.model,
         inputHash: summaries.inputHash,
+        promptVersion: summaries.promptVersion,
         status: summaries.status,
         batchId: summaries.batchId,
         error: summaries.error,
@@ -165,7 +168,7 @@ export function applyTargetUpdates(db: Db, repoId: number, updates: readonly Tar
 
             if (u.templateText){
                 const result = tx.update(summaries).set({
-                    tagline: u.templateText.tagline, summary: u.templateText.summary, model: "template",
+                    tagline: u.templateText.tagline, summary: u.templateText.summary, model: "template", promptVersion: null,
                     inputHash: u.inputHash, status: "skipped", error: null, updatedAt: now,
                 }).where(row).run();
                 if (result.changes > 0) changed = true;
@@ -217,11 +220,11 @@ export function loadBatchRows(db: Db, batchId: string): { id: number; kind: Summ
 
 /**
  * Writes answers back and returns how many rows changed.
- * - A summary lands as "fresh" with its text and inputHash. Its old vector is deleted, so
- *   the embeddings stage redoes it.
- * - A failure becomes "failed" and keeps any older text on show. A failure from the regular API
- *   leaves a batch's claim on the row in place: that request is already paid for, and its
- *   answer can still land.
+ * - A summary lands as "fresh" with its text, inputHash and prompt version. Its old vector is
+ *   deleted, so the embeddings stage redoes it.
+ * - A failure becomes "failed" and keeps any older text on show, with the prompt version that
+ *   wrote it. A failure from the regular API leaves a batch's claim on the row in place: that
+ *   request is already paid for, and its answer can still land.
  * - An answer from a batch that no longer owns the row (the row was redone some other way
  *   meanwhile) is dropped.
  */
@@ -234,7 +237,10 @@ export function writeResults(db: Db, results: readonly SummaryResult[]): number
         for (const r of results){
             const owner = r.batchId === null ? eq(summaries.id, r.id) : and(eq(summaries.id, r.id), eq(summaries.batchId, r.batchId));
             const answer = r.outcome.ok
-                ? { tagline: r.outcome.text.tagline, summary: r.outcome.text.summary, model: r.outcome.model, inputHash: r.inputHash, status: "fresh" as const, error: null }
+                ? {
+                    tagline: r.outcome.text.tagline, summary: r.outcome.text.summary, model: r.outcome.model, promptVersion: r.outcome.promptVersion,
+                    inputHash: r.inputHash, status: "fresh" as const, error: null,
+                }
                 : { status: "failed" as const, error: r.outcome.error };
 
             // Only a regular-API failure keeps the row's claim: a regular-API success supersedes the batch,

package/lib/engine/describe.ts

@@ -6,6 +6,7 @@ import { requireRepository } from "../db/store/structure";
 import { readIndexVersion } from "../db/store/version";
 import { DEFAULT_MAX_FILE_BYTES } from "../index/discover";
 import { repoKey } from "../index/paths";
+import { writtenInOlderStyle } from "../index/summarize/prompts";
 import { docsAbout, docsToReadFirst } from "./docs";
 import { stalePaths } from "./freshness";
 import { cachedAreaCounts, type TargetKind } from "./graph";
@@ -72,6 +73,7 @@ function describeUncached(db: Db, repo: string, repoId: number, version: number,
         ...(row?.tagline ? { tagline: row.tagline, summary: row.summary ?? "" } : {}),
         summaryStatus,
         ...(summaryStatus === "failed" && row?.error ? { summaryError: row.error } : {}),
+        ...(row && writtenInOlderStyle(row) ? { summaryOlderStyle: true as const } : {}),
         ...(docs.length > 0 ? { docs } : {}),
         stats: folder
             ? { fileCount: folder.fileCount, lineCount: sumLinesInside(db, repoId, targetPath) }

package/lib/index/summarize/budget.ts

@@ -16,8 +16,11 @@ export const ASSUMED_TOKENS: Readonly<Record<SummaryTargetKind, TokenUsage>> = {
     repo: { inputTokens: 1_000, outputTokens: 250 },
 };
 
-/** One request the projection prices: the kind of target it's for, and the model that answers it. */
-export type ProjectedRequest = { kind: SummaryTargetKind; model: string };
+/**
+ * One request the projection prices: the kind of target it's for, and the model that answers it. restyle: it's sent
+ * only because an older prompt version wrote its summary (PlannedTarget.restyle), so the estimate can say how many are.
+ */
+export type ProjectedRequest = { kind: SummaryTargetKind; model: string; restyle?: true };
 
 type Billing = "direct" | "batch";
 

package/lib/index/summarize/claude.ts

@@ -14,7 +14,7 @@ export const SUMMARY_SCHEMA = {
     type: "object",
     properties: {
         tagline: { type: "string", description: "A noun phrase of 3 to 6 plain words." },
-        summary: { type: "string", description: "2 to 4 sentences." },
+        summary: { type: "string", description: "Short paragraphs separated by a blank line, about the length the request gives." },
     },
     required: ["tagline", "summary"],
     additionalProperties: false,
@@ -32,6 +32,20 @@ const SENTENCE_END = /[.!?]["'`)\]]*$/;
 /** Two or more periods at the very end ("…in one place.."), which read as one. */
 const TRAILING_PERIODS = /\.{2,}$/;
 
+/** A paragraph break: two line breaks with only whitespace between them, CRLF and extra blank lines included. */
+const PARAGRAPH_BREAK = /\n\s*\n/;
+
+/**
+ * A summary's paragraphs in the one shape every reader splits on (P5): exactly one blank line between paragraphs,
+ * each trimmed, and a single line break inside one read as a space.
+ */
+function normalizeParagraphs(summary: string): string {
+    return summary.split(PARAGRAPH_BREAK)
+        .map((paragraph) => paragraph.trim().replace(/\s*\n\s*/g, " "))
+        .filter((paragraph) => paragraph !== "")
+        .join("\n\n");
+}
+
 /** Regular-API retries of a batch's cut-off answers in flight at once: as many as the runner's regular-API waves send. */
 const RETRY_CONCURRENCY = 4;
 
@@ -170,10 +184,11 @@ export function parseSummary(reply: Pick<Reply, "stop_reason" | "content">): Sum
     const tagline = cleanTagline(parsed.data.tagline);
     if (!tagline) throw new Error("the reply didn't have a tagline and a summary");
 
-    // ".." is a slip, not an ellipsis, and so is "...": the run becomes one "." before the check below
-    const summary = parsed.data.summary.replace(TRAILING_PERIODS, ".");
+    // ".." is a slip, not an ellipsis, and so is "...": the run becomes one "." before the check below.
+    // Then the paragraphs take their one shape, whatever line breaks the model wrote between them.
+    const summary = normalizeParagraphs(parsed.data.summary.replace(TRAILING_PERIODS, "."));
     // A normal stop doesn't prove the answer is whole: Summary Review #3 found well-formed JSON, stopped at end_turn,
-    // whose summary ended "…applyEdits. The". A summary is 2 to 4 sentences, so a whole one ends a sentence.
+    // whose summary ended "…applyEdits. The". A summary's paragraphs are whole sentences, so a whole one ends a sentence.
     if (!SENTENCE_END.test(summary)) throw new Error(`${CUT_OFF} (stop reason: end_turn): ${summary === "" ? "it's empty" : "it ends mid-sentence"}`);
     return { tagline, summary };
 }

package/lib/index/summarize/compare.ts

@@ -2,7 +2,7 @@ import type { FileFacts } from "../../db/store/summaries";
 import { ASSUMED_TOKENS } from "./budget";
 import { writtenByModel } from "./plan";
 import { costUsd, formatUsd, pricePerMTok, type PriceTable } from "./pricing";
-import { plural, PROMPT_VERSION } from "./prompts";
+import { joinParagraphs, plural, PROMPT_VERSION } from "./prompts";
 import type { SummaryOutcome } from "./types";
 
 /** Folder inputs, assumed for the estimate as the spend cap assumes them (budget.ts): a folder lists its children's taglines and summaries. */
@@ -108,7 +108,7 @@ export function renderComparison(report: ComparisonReport): string {
         for (const model of report.models){
             const outcome = row.outcomes[model];
             lines.push(outcome?.ok
-                ? `- ${answerLabel(model, outcome.model)} *${outcome.text.tagline}* — ${outcome.text.summary}`
+                ? `- ${answerLabel(model, outcome.model)} *${outcome.text.tagline}* — ${joinParagraphs(outcome.text.summary)}`
                 : `- **${model}:** failed: ${outcome ? outcome.error : "no answer"}`);
         }
     });

package/lib/index/summarize/plan.ts

@@ -4,12 +4,14 @@ import { targetKey, type FileFacts, type FolderFacts, type StoredSummary, type S
 import { DEFAULT_MAX_FILE_BYTES } from "../discover";
 import { dirOf } from "../paths";
 import {
-    importsOnlyFileSummary, PROMPT_VERSION, renderFolderPrompt, renderRepoPrompt, testFileSummary, testFolderSummary, type FolderItem,
+    folderTier, importsOnlyFileSummary, MAX_FOLDER_ITEMS, PROMPT_VERSION, renderFolderPrompt, renderRepoPrompt, testFileSummary,
+    testFolderSummary, UNRECORDED_PROMPT_VERSION, withLengthLine, writtenInOlderStyle, type FolderItem,
 } from "./prompts";
 
 /**
- * current: the text matches the current input · template: a test file or test-only folder whose
- * template is missing or out of date, written right away · ready: needs a model and can go now ·
+ * current: the text matches the current input, or an older prompt wrote it for this same input (P1) ·
+ * template: a test file or test-only folder whose template is missing or out of date, written right away ·
+ * ready: needs a model and can go now ·
  * waiting: needs a model, but a child isn't done yet · lazy: an out-of-date folder left
  * for later (redone when someone opens it) · in-flight: a batch already carries it ·
  * failed: failed earlier in this run; the next run retries it
@@ -26,10 +28,15 @@ export type PlannedTarget = {
     /** The row's status from now on; null leaves it as it is. */
     status: StoredSummaryStatus | null;
     templateText?: SummaryText;
-    /** Folders and the repo. A file's prompt is built from disk when it's sent. */
+    /** Folders and the repo, ending with the length line. A file's prompt is built from disk when it's sent. */
     prompt?: string;
+    /** planned to be written only because an older prompt version wrote its text (P1) */
+    restyle?: true;
 };
 
+/** Which summaries an older prompt wrote a plan restyles (PlanOptions.restyle). */
+export type Restyle = "all" | "none" | { path: string };
+
 export type PlanOptions = {
     models: { readonly file: string; readonly folder: string; readonly repo: string };
     /** true redoes out-of-date folders now (a full run); false leaves them lazy until opened (incremental runs). */
@@ -41,6 +48,19 @@ export type PlanOptions = {
      * cap, DEFAULT_MAX_FILE_BYTES, the product's only one; tests pass another to match an index they built with it.
      */
     maxFileBytes?: number;
+    /**
+     * which summaries an older prompt version wrote, with their input otherwise unchanged, are redone in the current
+     * style: every one (a full run and its estimate), the one a panel opened (summarizeNow), or none (the automatic jobs
+     * and the stage refresh). Default: "none".
+     */
+    restyle?: Restyle;
+    /**
+     * true plans an out-of-date folder or the repo whose text an older prompt version wrote, and which this plan doesn't
+     * restyle, as lazy, so it counts as settled and its parent can go over its older text. Under Q2 B only opening it
+     * rewrites it, so without this its parent would wait for it for good (the Task 4 review's I1, ruled as option a).
+     * summarizeNow and the sweep pass it. A full run and the stage refresh don't. Default: false.
+     */
+    olderFoldersWaitToBeOpened?: boolean;
 };
 
 /**
@@ -82,6 +102,10 @@ export function planSummaries(
         return row?.tagline && row.summary !== null ? { tagline: row.tagline, summary: row.summary } : null;
     };
 
+    // whether this plan restyles the target at `path`: a file and a folder never share a path, and only the repo's is ""
+    const restyled = (path: string): boolean =>
+        options.restyle === "all" || (typeof options.restyle === "object" && options.restyle.path === path);
+
     // ---- files: independent of each other ---------------------------------------------
     for (const file of files){
         const key = targetKey("file", file.path);
@@ -99,9 +123,10 @@ export function planSummaries(
         }
 
         // A file's prompt is built from its bytes, so its content hash stands in for the prompt
-        // here: same bytes, same prompt. Planning then never has to read source files.
-        const hash = inputHash(options.models.file, PROMPT_VERSION, `file\n${file.path}\n${file.contentHash}`);
-        planned.set(key, decide("file", file.path, options.models.file, hash, row, true, options));
+        // here: same bytes, same prompt. Planning then never has to read source files. Older
+        // prompt versions hashed the same stand-in.
+        const input = `file\n${file.path}\n${file.contentHash}`;
+        planned.set(key, decide("file", file.path, options.models.file, { input, olderInput: () => input, restyled: restyled(file.path) }, row, true, options));
     }
 
     // ---- folders, deepest first, so every child is planned before its parent ------------
@@ -164,11 +189,21 @@ export function planSummaries(
         }
 
         const model = kind === "repo" ? options.models.repo : options.models.folder;
-        const prompt = kind === "repo"
-            ? renderRepoPrompt({ name: folder.name, fileCount: folder.fileCount, folderCount: folders.length }, items)
-            : renderFolderPrompt(folder, items);
+        const repo = { name: folder.name, fileCount: folder.fileCount, folderCount: folders.length };
+        // The body is hashed without the length line (P2): the line follows from the file count, which the body
+        // holds, and from the prompt version, which the hash holds. The prompt sent ends with it.
+        const body = kind === "repo" ? renderRepoPrompt(repo, items) : renderFolderPrompt(folder, items);
+        // the body as prompt v2 rendered it, each child's summary verbatim, for the older check (P2): v2 is the only
+        // older version that stored summaries
+        const olderInput = (): string => (kind === "repo"
+            ? renderRepoPrompt(repo, items, MAX_FOLDER_ITEMS, false)
+            : renderFolderPrompt(folder, items, MAX_FOLDER_ITEMS, false));
+        const prompt = withLengthLine(body, kind, kind === "repo" ? "large" : folderTier(folder.fileCount));
         const key = targetKey(kind, folder.path);
-        planned.set(key, { ...decide(kind, folder.path, model, inputHash(model, PROMPT_VERSION, prompt), storedByKey.get(key), settled, options), prompt });
+        planned.set(key, {
+            ...decide(kind, folder.path, model, { input: body, olderInput, restyled: restyled(folder.path) }, storedByKey.get(key), settled, options),
+            prompt,
+        });
     }
 
     // ---- queue order --------------------------------------------------------------------
@@ -216,28 +251,53 @@ export function foldersAbove(targets: readonly PlannedTarget[], below: readonly
     return targets.filter((t) => above.has(targetKey(t.kind, t.path)));
 }
 
-/** The state of one model-written target, given its stored row and whether its children are settled. */
+/** What decide() hashes for one model-written target, and whether the plan restyles it. */
+type TargetInput = {
+    /** The input under the current prompt version: a file's stand-in, or a folder's or the repo's prompt body. */
+    input: string;
+    /** The input as an older prompt version rendered it (P2), rendered only when a row an older prompt wrote missed input's hash. */
+    olderInput: () => string;
+    /** Whether PlanOptions.restyle names this target. */
+    restyled: boolean;
+};
+
+/**
+ * The state of one model-written target, given its input, its stored row and whether its children are settled. A row
+ * an older prompt wrote for the same input is older style only (P1): current, unless the plan restyles it. One it wrote
+ * for another input is out of date, and with olderFoldersWaitToBeOpened a folder or the repo like that is lazy unless
+ * the plan restyles it.
+ */
 function decide(
     kind: SummaryTargetKind,
     path: string,
     model: string,
-    hash: string,
+    { input, olderInput, restyled }: TargetInput,
     row: StoredSummary | undefined,
     childrenSettled: boolean,
     options: PlanOptions
 ): PlannedTarget {
+    const hash = inputHash(model, PROMPT_VERSION, input);
     const base = { kind, path, model, inputHash: hash };
 
     if (row?.batchId) return { ...base, state: "in-flight", status: null };
     if (options.failedThisRun.has(targetKey(kind, path))) return { ...base, state: "failed", status: null };
     // the text on the row was written for exactly this input (an undone edit lands here too)
     if (row?.inputHash === hash) return { ...base, state: "current", status: row.status === "fresh" ? null : "fresh" };
+    // older style only (P1): a model wrote it with an older prompt, and its input hasn't changed since
+    if (row && writtenInOlderStyle(row) && row.inputHash === inputHash(model, row.promptVersion ?? UNRECORDED_PROMPT_VERSION, olderInput())){
+        const status = row.status === "fresh" ? null : "fresh";
+        if (!restyled) return { ...base, state: "current", status };
+        if (kind !== "file" && !options.refreshStaleFolders) return { ...base, state: "lazy", status, restyle: true };
+        return { ...base, state: childrenSettled ? "ready" : "waiting", status, restyle: true };
+    }
 
     const hasText = Boolean(row?.tagline);
     const status: StoredSummaryStatus = hasText ? "stale" : "pending";
     const keep = (state: TargetState): PlannedTarget => ({ ...base, state, status: row?.status === status ? null : status });
 
     if (kind !== "file" && hasText && !options.refreshStaleFolders) return keep("lazy");
+    // left to be opened (Q2 B, PlanOptions.olderFoldersWaitToBeOpened): only the target a plan restyles goes now
+    if (kind !== "file" && hasText && options.olderFoldersWaitToBeOpened && !restyled && row && writtenInOlderStyle(row)) return keep("lazy");
     return keep(childrenSettled ? "ready" : "waiting");
 }
 

package/lib/index/summarize/preview.ts

@@ -12,7 +12,7 @@ import { DEFAULT_SUMMARY_MODELS, type ResolvedSummaryModels } from "./models";
 import { planSummaries, writtenByModel, type PlannedTarget } from "./plan";
 import { formatUsd, pricePerMTok, type PriceTable } from "./pricing";
 import {
-    FILE_SYSTEM_PROMPT, FOLDER_SYSTEM_PROMPT, importerLine, MAX_FOLDER_ITEMS, plural, PROMPT_VERSION, REPO_SYSTEM_PROMPT,
+    FILE_SYSTEM_PROMPT, FOLDER_SYSTEM_PROMPT, importerLine, joinParagraphs, MAX_FOLDER_ITEMS, plural, PROMPT_VERSION, REPO_SYSTEM_PROMPT,
 } from "./prompts";
 import { buildFilePrompt, requireStructureReady } from "./run";
 import type { Summarizer, SummaryOutcome } from "./types";
@@ -95,10 +95,9 @@ export function nextWave(plan: readonly PlannedTarget[], scope: SampleScope, sen
 }
 
 /**
- * An answer as the row the real run would write for it (writeResults): its text and model, the planned inputHash, and
- * status "fresh". The planner reads only the hash, the batch id and the text, so it now calls the target current and
- * shows this text in its folder's prompt. A row the index already had keeps its id; a new one has none (0), since
- * it's never written. Pure.
+ * An answer as the row the real run would write for it (writeResults): its text and model, the current prompt version,
+ * the planned inputHash, and status "fresh". The planner then calls the target current and shows this text in its
+ * folder's prompt. A row the index already had keeps its id; a new one has none (0), since it's never written. Pure.
  */
 export function answeredRow(target: PlannedTarget, outcome: Extract<SummaryOutcome, { ok: true }>, previous: StoredSummary | undefined): StoredSummary {
     return {
@@ -109,6 +108,7 @@ export function answeredRow(target: PlannedTarget, outcome: Extract<SummaryOutco
         summary: outcome.text.summary,
         model: outcome.model,
         inputHash: target.inputHash,
+        promptVersion: PROMPT_VERSION,
         status: "fresh",
         batchId: null,
         error: null,
@@ -175,8 +175,11 @@ export async function runFolderPreview(input: PreviewInput): Promise<FolderPrevi
     };
 
     for (;;){
-        // refreshStaleFolders: true, as in a full run (npm run summarize): an out-of-date folder is redone, not left lazy
-        const plan = planSummaries(input.files, input.folders, [...rows.values()], { models: setting.models, refreshStaleFolders: true, failedThisRun });
+        // refreshStaleFolders: true and restyle: "all", as in a full run (npm run summarize): an out-of-date folder is
+        // redone, not left lazy, and so is a summary an older prompt version wrote
+        const plan = planSummaries(input.files, input.folders, [...rows.values()], {
+            models: setting.models, refreshStaleFolders: true, failedThisRun, restyle: "all",
+        });
         const wave = nextWave(plan, scope, sent);
 
         if (wave.length === 0){
@@ -433,10 +436,10 @@ function shown(preview: FolderPreview, kind: SummaryTargetKind, targetPath: stri
         if (!outcome.ok) return `failed: ${outcome.error}`;
         // a server-side fallback's answer isn't the summary the model asked would have written, and the reader should know
         const by = outcome.model === target?.model ? "" : ` (answered by ${outcome.model})`;
-        return `*${outcome.text.tagline}* — ${outcome.text.summary}${by}`;
+        return `*${outcome.text.tagline}* — ${joinParagraphs(outcome.text.summary)}${by}`;
     }
 
     const row = preview.rows.get(key);
-    if (target?.state === "current" && row?.tagline) return `*${row.tagline}* — ${row.summary} (already in the index, not sent again)`;
+    if (target?.state === "current" && row?.tagline) return `*${row.tagline}* — ${joinParagraphs(row.summary ?? "")} (already in the index, not sent again)`;
     return `not summarized (${target?.state ?? "not planned"})`;
 }