@sdods/db 0.2.0

package/dist/ingest/finalize.js

@@ -0,0 +1,132 @@
+import { enc, nowIso, readBool, readJson, readTs } from '../col.js';
+import { bumpLocatorStats, recomputeFlakyStats } from '../repos/improvement.js';
+import { emptyTotals } from './types.js';
+/**
+ * Derive scenario outcomes from attempts (count scenarios, not attempts), run totals/status,
+ * flaky stats windows and locator stats. Safe to re-run.
+ */
+export async function finalizeRun(input) {
+    const { adb, runId, projectId } = input;
+    const { db, driver } = adb;
+    const scenarios = await db
+        .selectFrom('scenarios')
+        .selectAll()
+        .where('run_id', '=', runId)
+        .execute();
+    const attempts = await db
+        .selectFrom('scenario_attempts')
+        .selectAll()
+        .where('run_id', '=', runId)
+        .execute();
+    const heals = await db
+        .selectFrom('heal_events')
+        .select(['scenario_id'])
+        .where('run_id', '=', runId)
+        .execute();
+    const healedScenarios = new Set(heals.map((h) => h.scenario_id).filter(Boolean));
+    const totals = emptyTotals();
+    let minStart = null;
+    let maxEnd = null;
+    for (const s of scenarios) {
+        const mine = attempts
+            .filter((a) => a.scenario_id === s.id)
+            .sort((a, b) => a.attempt - b.attempt);
+        const last = mine[mine.length - 1];
+        const finalStatus = last?.status ?? s.status ?? 'unknown';
+        const earlierFailed = mine
+            .slice(0, -1)
+            .some((a) => a.status === 'failed' || a.status === 'timedOut');
+        const flaky = mine.length > 1 && finalStatus === 'passed' && earlierFailed;
+        const startedAt = mine[0] ? readTs(mine[0].started_at) : readTs(s.started_at);
+        const finishedAt = last ? readTs(last.finished_at) : readTs(s.finished_at);
+        await db
+            .updateTable('scenarios')
+            .set({
+            status: finalStatus,
+            attempts_count: mine.length,
+            flaky: enc.bool(driver, flaky),
+            duration_ms: last?.duration_ms ?? s.duration_ms ?? null,
+            error_message: finalStatus === 'passed' ? null : (last?.error_message ?? s.error_message ?? null),
+            error_stack: finalStatus === 'passed' ? null : (last?.error_stack ?? s.error_stack ?? null),
+            started_at: startedAt,
+            finished_at: finishedAt,
+            updated_at: nowIso(),
+        })
+            .where('id', '=', s.id)
+            .execute();
+        totals.total++;
+        if (flaky)
+            totals.flaky++;
+        else if (finalStatus === 'passed')
+            totals.passed++;
+        else if (finalStatus === 'failed')
+            totals.failed++;
+        else if (finalStatus === 'timedOut')
+            totals.timedOut++;
+        else
+            totals.skipped++;
+        if (healedScenarios.has(s.id))
+            totals.healed++;
+        if (startedAt)
+            minStart =
+                minStart === null ? Date.parse(startedAt) : Math.min(minStart, Date.parse(startedAt));
+        if (finishedAt)
+            maxEnd = maxEnd === null ? Date.parse(finishedAt) : Math.max(maxEnd, Date.parse(finishedAt));
+    }
+    const started = input.runStartedAt ? Date.parse(input.runStartedAt) : minStart;
+    const finished = input.runFinishedAt ? Date.parse(input.runFinishedAt) : maxEnd;
+    totals.durationMs =
+        started !== null && finished !== null && started !== undefined && finished !== undefined
+            ? Math.max(0, finished - started)
+            : 0;
+    const status = totals.failed + totals.timedOut > 0 ? 'failed' : totals.total > 0 ? 'passed' : 'error';
+    const existing = await db
+        .selectFrom('runs')
+        .select(['totals_json'])
+        .where('id', '=', runId)
+        .executeTakeFirst();
+    const prev = readJson(existing?.totals_json) ?? {};
+    const files = new Set([
+        ...(Array.isArray(prev.ingested_files) ? prev.ingested_files : []),
+        ...input.ingestedFiles,
+    ]);
+    await db
+        .updateTable('runs')
+        .set({
+        status,
+        totals_json: enc.json({
+            ...prev,
+            ...totals,
+            ingested_files: [...files],
+            ingestedAt: nowIso(),
+        }),
+        shards_ingested: files.size,
+        started_at: started !== null && started !== undefined ? new Date(started).toISOString() : null,
+        finished_at: finished !== null && finished !== undefined ? new Date(finished).toISOString() : null,
+        duration_ms: totals.durationMs,
+        exit_code: input.exitCode ?? (status === 'passed' ? 0 : 1),
+        updated_at: nowIso(),
+    })
+        .where('id', '=', runId)
+        .execute();
+    // flaky stats over the window for every scenario key touched by this run
+    const keys = new Set(scenarios.map((s) => `${s.fingerprint}|${s.runner_project}`));
+    for (const key of keys) {
+        const [fingerprint, runnerProject] = key.split('|');
+        await recomputeFlakyStats(db, driver, { projectId, fingerprint, runnerProject });
+    }
+    // locator stats
+    for (const [selector, c] of input.locatorCounters) {
+        await bumpLocatorStats(db, projectId, {
+            selector,
+            fails: c.fails,
+            heals: c.heals,
+            uses: c.uses,
+            lastStrategy: c.lastStrategy ?? null,
+            suggestedSelector: c.suggested ?? null,
+        });
+    }
+    void readBool;
+    return { totals, status };
+}
+//# sourceMappingURL=finalize.js.map

package/dist/ingest/index.d.ts

@@ -0,0 +1,26 @@
+import type { RunManifest } from '@sdods/contracts/types';
+import type { SdodsDb } from '../create-db.js';
+import type { IngestResult, IngestRunOptions } from './types.js';
+export * from './types.js';
+export * from './parse-ndjson.js';
+export * from './message-index.js';
+export * from './attachments.js';
+export * from './cucumber-ingest.js';
+export * from './runner-json-ingest.js';
+export * from './finalize.js';
+/** Read `<runDir>/run.json` when present. */
+export declare function readManifest(runDir: string): RunManifest | null;
+/**
+ * Discover result files inside a run directory. Run directories outlive releases, so the
+ * pre-rename `pw-results*.json` name is still accepted alongside `runner-results*.json`.
+ */
+export declare function discoverRunFiles(runDir: string): {
+    ndjson: string[];
+    runnerJson: string[];
+};
+/**
+ * Ingest one run (all its NDJSON shards and runner JSON files) into the database.
+ * Idempotent: natural keys converge on re-run; `replace` wipes the run's children first.
+ */
+export declare function ingestRun(adb: SdodsDb, opts: IngestRunOptions): Promise<IngestResult>;
+//# sourceMappingURL=index.d.ts.map

package/dist/ingest/index.js

@@ -0,0 +1,193 @@
+import { existsSync, readFileSync, readdirSync } from 'node:fs';
+import { basename, join, resolve } from 'node:path';
+import { runFiles } from '@sdods/contracts/names';
+import { readJson } from '../col.js';
+import { ensureProject, getProjectBySlug } from '../repos/projects.js';
+import { deleteRunChildren, upsertRun } from '../repos/runs.js';
+import { IngestSession } from './cucumber-ingest.js';
+import { finalizeRun } from './finalize.js';
+import { fileSha256, parseNdjson } from './parse-ndjson.js';
+import { ingestRunnerJson } from './runner-json-ingest.js';
+export * from './types.js';
+export * from './parse-ndjson.js';
+export * from './message-index.js';
+export * from './attachments.js';
+export * from './cucumber-ingest.js';
+export * from './runner-json-ingest.js';
+export * from './finalize.js';
+/** Read `<runDir>/run.json` when present. */
+export function readManifest(runDir) {
+    const file = join(runDir, runFiles.manifest);
+    if (!existsSync(file))
+        return null;
+    try {
+        return JSON.parse(readFileSync(file, 'utf8'));
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Discover result files inside a run directory. Run directories outlive releases, so the
+ * pre-rename `pw-results*.json` name is still accepted alongside `runner-results*.json`.
+ */
+export function discoverRunFiles(runDir) {
+    const ndjson = [];
+    const runnerJson = [];
+    if (!existsSync(runDir))
+        return { ndjson, runnerJson };
+    for (const f of readdirSync(runDir)) {
+        if (/^messages(\.shard-\d+)?\.ndjson$/.test(f))
+            ndjson.push(join(runDir, f));
+        if (/^(runner|pw)-results(\.shard-\d+)?\.json$/.test(f))
+            runnerJson.push(join(runDir, f));
+    }
+    return { ndjson: ndjson.sort(), runnerJson: runnerJson.sort() };
+}
+/**
+ * Ingest one run (all its NDJSON shards and runner JSON files) into the database.
+ * Idempotent: natural keys converge on re-run; `replace` wipes the run's children first.
+ */
+export async function ingestRun(adb, opts) {
+    const artifactsRoot = resolve(opts.artifactsRoot);
+    const runDir = join(artifactsRoot, opts.runId);
+    const manifest = opts.manifest === undefined ? readManifest(runDir) : opts.manifest;
+    const projectSlug = manifest?.projectSlug ?? opts.projectSlug;
+    if (!projectSlug)
+        throw new Error('ingestRun needs a project slug (manifest run.json or --project).');
+    const discovered = discoverRunFiles(runDir);
+    const ndjsonPaths = (opts.ndjsonPaths?.length ? opts.ndjsonPaths : discovered.ndjson).map((p) => resolve(p));
+    const runnerJsonPaths = (opts.runnerJsonPaths?.length ? opts.runnerJsonPaths : discovered.runnerJson).map((p) => resolve(p));
+    if (ndjsonPaths.length === 0 && runnerJsonPaths.length === 0)
+        throw new Error(`No messages*.ndjson or runner-results*.json found for run ${opts.runId} under ${runDir}.`);
+    const { db, driver } = adb;
+    const project = (await getProjectBySlug(db, projectSlug)) ?? {
+        id: await ensureProject(db, driver, projectSlug),
+        workspaceId: null,
+    };
+    const ctx = {
+        adb,
+        runId: opts.runId,
+        projectId: project.id,
+        projectSlug,
+        artifactsRoot,
+        runDir,
+        workspaceId: project.workspaceId ?? null,
+    };
+    const existingRun = await db
+        .selectFrom('runs')
+        .select(['totals_json'])
+        .where('id', '=', opts.runId)
+        .executeTakeFirst();
+    const prevTotals = readJson(existingRun?.totals_json) ?? {};
+    const alreadyIngested = new Set(Array.isArray(prevTotals.ingested_files) ? prevTotals.ingested_files : []);
+    await upsertRun(db, driver, {
+        id: opts.runId,
+        projectId: project.id,
+        workspaceId: ctx.workspaceId,
+        envName: manifest?.env ?? opts.env ?? 'unknown',
+        trigger: manifest?.trigger ?? opts.trigger ?? 'cli',
+        status: 'running',
+        suiteTag: manifest?.suiteTag ?? null,
+        tagsExpr: manifest?.tagsExpr ?? null,
+        layers: manifest?.layers ?? [],
+        browsers: manifest?.browsers ?? [],
+        shardTotal: manifest?.shardTotal ?? null,
+        gitSha: manifest?.git?.sha ?? null,
+        gitBranch: manifest?.git?.branch ?? null,
+        ciProvider: manifest?.ci?.provider ?? null,
+        ciRunId: manifest?.ci?.runId ?? null,
+        ciUrl: manifest?.ci?.url ?? null,
+        command: manifest?.command ?? null,
+        startedAt: manifest?.startedAt ?? null,
+        artifactsDir: runDir,
+        htmlReportRel: existsSync(join(runDir, runFiles.htmlReport)) ? runFiles.htmlReport : null,
+        exitCode: manifest?.exitCode ?? null,
+        process: manifest?.process ?? null,
+    });
+    if (opts.replace)
+        await deleteRunChildren(db, opts.runId);
+    const locatorCounters = new Map();
+    const filesIngested = [];
+    const filesSkipped = [];
+    let parseErrors = 0;
+    let runStartedAt = manifest?.startedAt ?? null;
+    let runFinishedAt = manifest?.finishedAt ?? null;
+    const counts = { scenarios: 0, attempts: 0, steps: 0, artifacts: 0, healEvents: 0 };
+    for (const file of ndjsonPaths) {
+        const sha = fileSha256(file);
+        const key = `${basename(file)}@${sha}`;
+        if (alreadyIngested.has(key) && !opts.replace) {
+            filesSkipped.push(file);
+            continue;
+        }
+        const session = new IngestSession(ctx);
+        const stats = { lines: 0, parsed: 0, skipped: 0 };
+        for await (const env of parseNdjson(file, stats))
+            await session.consume(env);
+        parseErrors += stats.skipped;
+        for (const [sel, c] of session.locatorCounters) {
+            const prev = locatorCounters.get(sel) ?? { fails: 0, heals: 0, uses: 0 };
+            locatorCounters.set(sel, {
+                fails: prev.fails + c.fails,
+                heals: prev.heals + c.heals,
+                uses: prev.uses + c.uses,
+                lastStrategy: c.lastStrategy ?? prev.lastStrategy,
+                suggested: c.suggested ?? prev.suggested,
+            });
+        }
+        counts.scenarios += session.counts.scenarios;
+        counts.attempts += session.counts.attempts;
+        counts.steps += session.counts.steps;
+        counts.artifacts += session.counts.artifacts;
+        counts.healEvents += session.counts.healEvents;
+        if (session.runStartedAt && (!runStartedAt || session.runStartedAt < runStartedAt))
+            runStartedAt = session.runStartedAt;
+        if (session.runFinishedAt && (!runFinishedAt || session.runFinishedAt > runFinishedAt))
+            runFinishedAt = session.runFinishedAt;
+        filesIngested.push(key);
+    }
+    for (const file of runnerJsonPaths) {
+        const sha = fileSha256(file);
+        const key = `${basename(file)}@${sha}`;
+        if (alreadyIngested.has(key) && !opts.replace) {
+            filesSkipped.push(file);
+            continue;
+        }
+        const r = await ingestRunnerJson(ctx, file, locatorCounters);
+        counts.scenarios += r.scenarios;
+        counts.attempts += r.attempts;
+        counts.steps += r.steps;
+        counts.artifacts += r.artifacts;
+        if (r.startedAt && (!runStartedAt || r.startedAt < runStartedAt))
+            runStartedAt = r.startedAt;
+        if (r.finishedAt && (!runFinishedAt || r.finishedAt > runFinishedAt))
+            runFinishedAt = r.finishedAt;
+        filesIngested.push(key);
+    }
+    const { totals, status } = await finalizeRun({
+        adb,
+        runId: opts.runId,
+        projectId: project.id,
+        locatorCounters,
+        runStartedAt,
+        runFinishedAt,
+        ingestedFiles: filesIngested,
+        exitCode: manifest?.exitCode ?? null,
+    });
+    return {
+        runId: opts.runId,
+        projectSlug,
+        totals,
+        status,
+        scenarios: counts.scenarios,
+        attempts: counts.attempts,
+        steps: counts.steps,
+        artifacts: counts.artifacts,
+        healEvents: counts.healEvents,
+        filesIngested,
+        filesSkipped,
+        parseErrors,
+    };
+}
+//# sourceMappingURL=index.js.map

package/dist/ingest/message-index.d.ts

@@ -0,0 +1,64 @@
+/**
+ * In-memory index of the static cucumber envelopes (gherkinDocument, pickle, stepDefinition, hook,
+ * testCase) that the dynamic ones (testCaseStarted, testStepFinished, attachment …) refer to.
+ */
+export interface GherkinStepInfo {
+    keyword: string;
+    line: number;
+}
+export interface PickleInfo {
+    id: string;
+    uri: string;
+    name: string;
+    tags: string[];
+    steps: Array<{
+        id: string;
+        text: string;
+        type?: string;
+        astNodeIds: string[];
+        argument?: unknown;
+    }>;
+    astNodeIds: string[];
+}
+export interface TestCaseInfo {
+    id: string;
+    pickleId: string;
+    testSteps: Array<{
+        id: string;
+        pickleStepId?: string;
+        hookId?: string;
+        stepDefinitionIds?: string[];
+    }>;
+}
+export declare class MessageIndex {
+    readonly docs: Map<string, any>;
+    readonly pickles: Map<string, PickleInfo>;
+    readonly testCases: Map<string, TestCaseInfo>;
+    readonly stepDefs: Map<string, any>;
+    readonly hooks: Map<string, any>;
+    /** ast node id → step keyword/line (from gherkin documents) */
+    private readonly astSteps;
+    /** ast node id (examples table row) → { index (1-based), examplesTitle } */
+    private readonly astExampleRows;
+    /** ast node id (scenario) → { name, line } */
+    private readonly astScenarios;
+    private readonly featureNameByUri;
+    addDoc(doc: any): void;
+    addPickle(p: any): void;
+    addStepDef(sd: any): void;
+    addHook(h: any): void;
+    addTestCase(tc: any): void;
+    featureName(uri: string): string | null;
+    stepKeyword(pickleStep: {
+        astNodeIds: string[];
+    }): string | null;
+    /** 1-based Examples row index for an outline pickle, null for plain scenarios. */
+    exampleIndex(pickle: PickleInfo): number | null;
+    scenarioLine(pickle: PickleInfo): number | null;
+    /** Base scenario name (without example values substituted) */
+    scenarioTitle(pickle: PickleInfo): string;
+    hookType(hookId: string): 'before' | 'after' | null;
+    hookName(hookId: string): string;
+    definitionLocation(ids: string[] | undefined): string | null;
+}
+//# sourceMappingURL=message-index.d.ts.map

package/dist/ingest/message-index.js

@@ -0,0 +1,146 @@
+export class MessageIndex {
+    docs = new Map();
+    pickles = new Map();
+    testCases = new Map();
+    stepDefs = new Map();
+    hooks = new Map();
+    /** ast node id → step keyword/line (from gherkin documents) */
+    astSteps = new Map();
+    /** ast node id (examples table row) → { index (1-based), examplesTitle } */
+    astExampleRows = new Map();
+    /** ast node id (scenario) → { name, line } */
+    astScenarios = new Map();
+    featureNameByUri = new Map();
+    addDoc(doc) {
+        if (!doc?.uri)
+            return;
+        this.docs.set(doc.uri, doc);
+        const feature = doc.feature;
+        if (!feature)
+            return;
+        this.featureNameByUri.set(doc.uri, feature.name ?? '');
+        const walkChildren = (children) => {
+            for (const child of children ?? []) {
+                if (child.rule)
+                    walkChildren(child.rule.children);
+                const node = child.background ?? child.scenario;
+                if (!node)
+                    continue;
+                if (child.scenario) {
+                    this.astScenarios.set(node.id, {
+                        name: node.name ?? '',
+                        line: node.location?.line ?? 0,
+                        tags: (node.tags ?? []).map((t) => t.name),
+                    });
+                }
+                for (const step of node.steps ?? []) {
+                    this.astSteps.set(step.id, {
+                        keyword: String(step.keyword ?? '').trim(),
+                        line: step.location?.line ?? 0,
+                    });
+                }
+                let rowIndex = 0;
+                for (const ex of node.examples ?? []) {
+                    for (const row of ex.tableBody ?? []) {
+                        rowIndex++;
+                        this.astExampleRows.set(row.id, { index: rowIndex, line: row.location?.line ?? 0 });
+                    }
+                }
+            }
+        };
+        walkChildren(feature.children);
+    }
+    addPickle(p) {
+        if (!p?.id)
+            return;
+        this.pickles.set(p.id, {
+            id: p.id,
+            uri: p.uri,
+            name: p.name ?? '',
+            tags: (p.tags ?? []).map((t) => t.name),
+            steps: (p.steps ?? []).map((s) => ({
+                id: s.id,
+                text: s.text ?? '',
+                type: s.type,
+                astNodeIds: s.astNodeIds ?? [],
+                argument: s.argument,
+            })),
+            astNodeIds: p.astNodeIds ?? [],
+        });
+    }
+    addStepDef(sd) {
+        if (sd?.id)
+            this.stepDefs.set(sd.id, sd);
+    }
+    addHook(h) {
+        if (h?.id)
+            this.hooks.set(h.id, h);
+    }
+    addTestCase(tc) {
+        if (!tc?.id)
+            return;
+        this.testCases.set(tc.id, { id: tc.id, pickleId: tc.pickleId, testSteps: tc.testSteps ?? [] });
+    }
+    featureName(uri) {
+        return this.featureNameByUri.get(uri) ?? null;
+    }
+    stepKeyword(pickleStep) {
+        for (const id of pickleStep.astNodeIds) {
+            const s = this.astSteps.get(id);
+            if (s)
+                return s.keyword;
+        }
+        return null;
+    }
+    /** 1-based Examples row index for an outline pickle, null for plain scenarios. */
+    exampleIndex(pickle) {
+        for (const id of pickle.astNodeIds) {
+            const row = this.astExampleRows.get(id);
+            if (row)
+                return row.index;
+        }
+        return null;
+    }
+    scenarioLine(pickle) {
+        for (const id of pickle.astNodeIds) {
+            const sc = this.astScenarios.get(id);
+            if (sc)
+                return sc.line;
+        }
+        return null;
+    }
+    /** Base scenario name (without example values substituted) */
+    scenarioTitle(pickle) {
+        for (const id of pickle.astNodeIds) {
+            const sc = this.astScenarios.get(id);
+            if (sc)
+                return sc.name || pickle.name;
+        }
+        return pickle.name;
+    }
+    hookType(hookId) {
+        const h = this.hooks.get(hookId);
+        if (!h)
+            return null;
+        const t = String(h.type ?? h.name ?? '').toLowerCase();
+        if (t.includes('after'))
+            return 'after';
+        if (t.includes('before'))
+            return 'before';
+        return null;
+    }
+    hookName(hookId) {
+        const h = this.hooks.get(hookId);
+        return h?.name ?? h?.sourceReference?.uri ?? 'hook';
+    }
+    definitionLocation(ids) {
+        for (const id of ids ?? []) {
+            const sd = this.stepDefs.get(id);
+            const ref = sd?.sourceReference;
+            if (ref?.uri)
+                return `${ref.uri}:${ref.location?.line ?? 0}`;
+        }
+        return null;
+    }
+}
+//# sourceMappingURL=message-index.js.map

package/dist/ingest/parse-ndjson.d.ts

@@ -0,0 +1,10 @@
+export type Envelope = Record<string, any>;
+export interface ParseStats {
+    lines: number;
+    parsed: number;
+    skipped: number;
+}
+/** Tolerant NDJSON reader: blank and corrupt lines are counted and skipped, never fatal. */
+export declare function parseNdjson(file: string, stats?: ParseStats): AsyncGenerator<Envelope>;
+export declare function fileSha256(file: string): string;
+//# sourceMappingURL=parse-ndjson.d.ts.map

package/dist/ingest/parse-ndjson.js

@@ -0,0 +1,35 @@
+import { createHash } from 'node:crypto';
+import { createReadStream, readFileSync } from 'node:fs';
+import { createInterface } from 'node:readline';
+/** Tolerant NDJSON reader: blank and corrupt lines are counted and skipped, never fatal. */
+export async function* parseNdjson(file, stats) {
+    const rl = createInterface({
+        input: createReadStream(file, { encoding: 'utf8' }),
+        crlfDelay: Infinity,
+    });
+    for await (const raw of rl) {
+        if (stats)
+            stats.lines++;
+        const line = raw.trim();
+        if (!line)
+            continue;
+        try {
+            const env = JSON.parse(line);
+            if (env && typeof env === 'object') {
+                if (stats)
+                    stats.parsed++;
+                yield env;
+            }
+            else if (stats)
+                stats.skipped++;
+        }
+        catch {
+            if (stats)
+                stats.skipped++;
+        }
+    }
+}
+export function fileSha256(file) {
+    return createHash('sha256').update(readFileSync(file)).digest('hex');
+}
+//# sourceMappingURL=parse-ndjson.js.map

package/dist/ingest/pw-json-ingest.d.ts

@@ -0,0 +1,14 @@
+import type { LocatorCounter } from './finalize.js';
+import { type IngestContext } from './types.js';
+export interface PwJsonCounts {
+    scenarios: number;
+    attempts: number;
+    steps: number;
+    artifacts: number;
+}
+/** Ingest a Playwright `json` reporter file (recorded layer / plain specs) into the same tables. */
+export declare function ingestPwJson(ctx: IngestContext, file: string, locatorCounters: Map<string, LocatorCounter>): Promise<PwJsonCounts & {
+    startedAt: string | null;
+    finishedAt: string | null;
+}>;
+//# sourceMappingURL=pw-json-ingest.d.ts.map