@jinn-network/jinn-layer 0.1.0

package/dist/distill-traces.js

@@ -0,0 +1,211 @@
+function uniq(values) {
+    return [...new Set([...values].filter((v) => typeof v === 'string' && v.trim() !== '').map((v) => v.trim()))];
+}
+function oneLine(value, max = 240) {
+    if (typeof value !== 'string')
+        return undefined;
+    const text = value.replace(/\s+/g, ' ').trim();
+    if (!text)
+        return undefined;
+    return text.length > max ? `${text.slice(0, max - 1)}…` : text;
+}
+function parseJsonObject(value) {
+    if (typeof value !== 'string')
+        return null;
+    try {
+        const parsed = JSON.parse(value);
+        return parsed && typeof parsed === 'object' && !Array.isArray(parsed)
+            ? parsed
+            : null;
+    }
+    catch {
+        return null;
+    }
+}
+function commandFromStep(step) {
+    const args = parseJsonObject(step.attributes['tool.args']);
+    return oneLine(args?.['command'] ?? args?.['cmd'] ?? step.attributes['command']);
+}
+function fileFromStep(step) {
+    const args = parseJsonObject(step.attributes['tool.args']);
+    return oneLine(step.attributes['file.path'] ?? args?.['path'] ?? args?.['file']);
+}
+function eventKind(step) {
+    return typeof step.attributes['jinn.capture.event.kind'] === 'string'
+        ? step.attributes['jinn.capture.event.kind']
+        : step.name;
+}
+function eventText(step) {
+    return oneLine(step.attributes['message.content'] ??
+        step.attributes['tool.result'] ??
+        step.attributes['file.diff'] ??
+        commandFromStep(step) ??
+        fileFromStep(step));
+}
+function toolName(step) {
+    return oneLine(step.attributes['tool.name']);
+}
+function traceId(task) {
+    return `local-capture:${task.session.sessionId}`;
+}
+export function traceCardFromCapture(task) {
+    const commands = uniq(task.steps.map(commandFromStep));
+    const filesTouched = uniq(task.steps.map(fileFromStep));
+    const errorSnippets = uniq(task.steps.map((step) => {
+        const isError = step.attributes['tool.is_error'] === true || step.attributes['tool.is_error'] === 'true';
+        return isError ? oneLine(step.attributes['tool.result']) : undefined;
+    }));
+    const toolNames = uniq([
+        ...task.environment.tools,
+        ...task.steps.map(toolName),
+    ]);
+    const redactionFlags = task.steps.some((step) => step.redactedKeys.length > 0) ? ['redacted'] : [];
+    return {
+        traceId: traceId(task),
+        sessionId: task.session.sessionId,
+        sourceTool: task.environment.harness.name,
+        startedAt: task.session.capturedAt,
+        durationMs: task.cost.durationMs,
+        summary: task.task.summary,
+        tools: toolNames,
+        commands,
+        filesTouched,
+        errorSnippets,
+        outcome: task.outcome.status,
+        skillsUsed: [],
+        redactionFlags,
+    };
+}
+function eventView(step, includeAttributes = false) {
+    const out = {
+        spanId: step.spanId,
+        kind: eventKind(step),
+        name: step.name,
+    };
+    const text = eventText(step);
+    const tool = toolName(step);
+    if (text)
+        out.text = text;
+    if (tool)
+        out.toolName = tool;
+    if (includeAttributes)
+        out.attributes = step.attributes;
+    return out;
+}
+export function readTrace(task, opts) {
+    const base = { mode: opts.mode, traceId: traceId(task), summary: task.task.summary };
+    if (opts.mode === 'summary')
+        return base;
+    if (opts.mode === 'full_transcript' && !opts.allowFullTranscript) {
+        throw new Error('full transcript reads require explicit allowFullTranscript=true');
+    }
+    const events = task.steps.map((step) => eventView(step, opts.mode === 'full_transcript'));
+    if (opts.mode === 'events' || opts.mode === 'full_transcript')
+        return { ...base, events };
+    if (opts.mode === 'tool_calls') {
+        return {
+            ...base,
+            toolCalls: events.filter((event) => event.kind === 'tool-call' || event.toolName !== undefined),
+        };
+    }
+    const q = opts.query?.toLowerCase();
+    const filtered = q
+        ? events.filter((event) => JSON.stringify(event).toLowerCase().includes(q))
+        : events;
+    return { ...base, events: filtered.slice(0, opts.limit ?? 20) };
+}
+function cardText(card) {
+    return [
+        card.summary,
+        ...card.tools,
+        ...card.commands,
+        ...card.filesTouched,
+        ...card.errorSnippets,
+        card.outcome,
+    ].join('\n').toLowerCase();
+}
+export function searchTraceCards(cards, query) {
+    const matches = cards.filter((card) => {
+        if (query.query && !cardText(card).includes(query.query.toLowerCase()))
+            return false;
+        if (query.command && !card.commands.some((c) => c.toLowerCase().includes(query.command.toLowerCase())))
+            return false;
+        if (query.file && !card.filesTouched.some((f) => f.toLowerCase().includes(query.file.toLowerCase())))
+            return false;
+        if (query.error && !card.errorSnippets.some((e) => e.toLowerCase().includes(query.error.toLowerCase())))
+            return false;
+        if (query.tool && !card.tools.some((t) => t.toLowerCase().includes(query.tool.toLowerCase())))
+            return false;
+        if (query.outcome && card.outcome !== query.outcome)
+            return false;
+        return true;
+    });
+    return matches
+        .sort((a, b) => b.startedAt.localeCompare(a.startedAt))
+        .slice(0, query.limit ?? matches.length);
+}
+function estimateTokens(text) {
+    return Math.ceil(new TextEncoder().encode(text).length / 4);
+}
+function costFor(tokens) {
+    if (tokens < 20_000)
+        return 'low';
+    if (tokens < 80_000)
+        return 'medium';
+    return 'high';
+}
+function grouped(cards, key) {
+    const out = new Map();
+    for (const card of cards) {
+        for (const value of card[key]) {
+            const normalized = value.toLowerCase();
+            const group = out.get(normalized) ?? [];
+            group.push(card);
+            out.set(normalized, group);
+        }
+    }
+    return out;
+}
+export function clusterTraceCards(cards) {
+    const candidates = [];
+    const addGroups = (label, key, groups, priority) => {
+        for (const [signal, group] of groups) {
+            if (group.length < 2)
+                continue;
+            const tokenEstimate = estimateTokens(JSON.stringify(group));
+            candidates.push({
+                title: `${label}: ${group[0]?.[key === 'errors' ? 'errorSnippets' : key === 'commands' ? 'commands' : 'filesTouched']
+                    .find((v) => v.toLowerCase() === signal) ?? signal}`,
+                traceIds: group.map((card) => card.traceId),
+                repeatedSignals: { [key]: [signal] },
+                evidenceTierEstimate: 'recurring-pattern',
+                estimatedSourceTokens: tokenEstimate,
+                estimatedDistillCost: costFor(tokenEstimate),
+                privacyFlags: uniq(group.flatMap((card) => card.redactionFlags)),
+                priority,
+            });
+        }
+    };
+    addGroups('Repeated error', 'errors', grouped(cards, 'errorSnippets'), 0);
+    addGroups('Repeated command', 'commands', grouped(cards, 'commands'), 1);
+    addGroups('Repeated file', 'files', grouped(cards, 'filesTouched'), 2);
+    if (candidates.length === 0 && cards.length === 1) {
+        const tokenEstimate = estimateTokens(JSON.stringify(cards[0]));
+        candidates.push({
+            title: cards[0].summary,
+            traceIds: [cards[0].traceId],
+            repeatedSignals: {},
+            evidenceTierEstimate: 'single-example',
+            estimatedSourceTokens: tokenEstimate,
+            estimatedDistillCost: costFor(tokenEstimate),
+            privacyFlags: cards[0].redactionFlags,
+            priority: 3,
+        });
+    }
+    return candidates
+        .sort((a, b) => b.traceIds.length - a.traceIds.length || a.priority - b.priority || a.title.localeCompare(b.title))
+        .map(({ priority, ...candidate }, index) => ({
+        candidateId: `local-candidate-${String(index + 1).padStart(3, '0')}`,
+        ...candidate,
+    }));
+}

package/dist/distill.d.ts

@@ -0,0 +1,180 @@
+/**
+ * The distillation step — evidence clusters → layer-2 skills
+ * (spec/2026-07-06-distillation-v1.md §7, D4/D10).
+ *
+ * Scripted, single-shot, flat (recursion/hierarchy are v3). Three modes keyed to
+ * the cluster's tier (§7): pattern → strategic-pattern skill; lesson →
+ * failure-lesson skill (diagnosis, not prescription); contrastive → one skill
+ * from both polarities of an instance. Each skill is:
+ *   1. distilled by an injected LLM port (`deps.distill`) using
+ *      `jinn-skill-distill-prompt-v1` (the port owns the model call);
+ *   2. **secret-scrubbed fail-closed** — the body is run through the layer-2
+ *      pipeline; if the scrub would change anything (a secret was present), the
+ *      skill is DROPPED, not published (so published bodies are provably clean
+ *      AND never re-defaced, §7 step 4 / §10);
+ *   3. **contamination-scanned** against the held-out slate (§12 axis 3) — a
+ *      body naming a slate instance/repo/PR is dropped;
+ *   4. **structurally gated** (§7 step 6, v0.5) — the fixed skeleton (five
+ *      non-empty sections), the description anti-trigger ("Not for:"), and (for
+ *      lessons) the imperative-counterfactual guard, all DETERMINISTIC (a deep
+ *      quality judge is deferred admission-checking, §13);
+ *   5. published via `publishSkill()` with provenance back-links, the prompt
+ *      SHA, and the auditability fields (distill model + token estimates, §5).
+ *
+ * Clustering is upstream (human-curated for v1, §7) — this consumes pre-formed
+ * clusters of eligible evidence.
+ */
+import { type ScrubPipeline } from '@jinn-network/core/scrub';
+import { type SkillPackage } from './skill-package.js';
+import type { MetaCluster } from './cluster.js';
+export interface DistillCluster {
+    clusterId: string;
+    /**
+     * `pattern` (successes), `lesson` (evaluator-confirmed failures), or
+     * `contrastive` (both polarities for one instance — the pass↔fail delta,
+     * §7 v0.5). A contrastive cluster carries evidence refs from BOTH polarities.
+     */
+    tier: 'pattern' | 'lesson' | 'contrastive';
+    /** Source evidence envelope refs → `metadata.jinn.provenance`. */
+    evidenceRefs: string[];
+    /** The distinct instance ids in the cluster (audit / dedup upstream). */
+    instanceIds: string[];
+    /** Opaque payload handed to the LLM port (the cluster's evidence). */
+    input: unknown;
+}
+export interface DistillLLMOutput {
+    name: string;
+    description: string;
+    body: string;
+}
+export interface DistillDeps {
+    /** The LLM port: runs jinn-skill-distill-prompt-v1 over the cluster. */
+    distill: (cluster: DistillCluster) => Promise<DistillLLMOutput>;
+    /** Publish a skill package as a corpus record (Plan A `publishSkill`). */
+    publishSkill: (pkg: SkillPackage) => Promise<{
+        envelopeRef: string;
+        anchorTx: string | null;
+    }>;
+    /** The held-out slate for the contamination scan (§12). */
+    slate: {
+        instanceIds: Set<string>;
+        repos?: Set<string>;
+    };
+    distribution?: string;
+    /** Evidence tier recorded on generated local skills. Defaults to evaluator-verified network posture. */
+    verifiabilityTier?: string;
+    /** Local lifecycle status for private generated skills. */
+    skillStatus?: 'experimental' | 'stable' | 'deprecated';
+    /** Local evidence tier for private generated skills. */
+    evidenceTier?: 'single-example' | 'recurring-pattern' | 'contrastive' | 'user-confirmed' | 'stable' | 'deprecated';
+    sourceTools?: string[];
+    targetTools?: string[];
+    /** The model that ran distillation — recorded in provenance (§5 auditability). */
+    distillModel?: string;
+    scrubPipeline?: ScrubPipeline;
+    now?: () => Date;
+    /** Progress seam (#1533): fired once before the loop with every cluster. */
+    onPlan?: (plan: ClusterPlanEntry[]) => void;
+    /** Progress seam (#1533): fired around each cluster's distill call. */
+    onCluster?: (ev: ClusterProgressEvent) => void;
+}
+/** One row of the up-front cluster plan handed to {@link DistillDeps.onPlan}. */
+export interface ClusterPlanEntry {
+    clusterId: string;
+    /** 1-based position in the run. */
+    index: number;
+    captureCount: number;
+    /** The cluster's evidence refs — a consumer can map these back to sources. */
+    refs: string[];
+}
+/** A start/end progress event around one cluster's distill call (#1533). */
+export interface ClusterProgressEvent {
+    phase: 'start' | 'end';
+    clusterId: string;
+    /** 1-based position in the run. */
+    index: number;
+    total: number;
+    /** End-phase only. */
+    outcome?: 'published' | 'rejected' | 'error';
+    skillName?: string;
+    reason?: string;
+    error?: string;
+    durationMs?: number;
+}
+export type SkillKind = 'strategic-pattern' | 'failure-lesson' | 'contrastive' | 'cross-instance';
+export interface DistillResult {
+    published: Array<{
+        clusterId: string;
+        skillKind: SkillKind;
+        envelopeRef: string;
+        pkg: SkillPackage;
+    }>;
+    rejected: Array<{
+        clusterId: string;
+        reason: string;
+    }>;
+    errors: Array<{
+        clusterId: string;
+        error: string;
+    }>;
+}
+/**
+ * Scan a distilled body for held-out slate tokens (instance id / repo / PR#).
+ * Delimiter-bearing tokens match as substrings; bare repo names match only on
+ * word boundaries, so a common-word repo does not silently drop a clean skill.
+ */
+export declare function lexicalContaminationScan(body: string, slate: {
+    instanceIds: Set<string>;
+    repos?: Set<string>;
+}): {
+    contaminated: boolean;
+    hits: string[];
+};
+export declare function distillClusters(clusters: DistillCluster[], deps: DistillDeps): Promise<DistillResult>;
+/** Stage-2 (cross-instance) LLM output: a distilled skill plus the source ids it corroborates. */
+export interface MetaDistillLLMOutput extends DistillLLMOutput {
+    /** Opaque source ids (s1, s2, …) the model says corroborate the emitted rule. */
+    supports: string[];
+}
+export interface MetaDistillDeps {
+    /** The meta LLM port: runs jinn-skill-meta-distill-prompt-v1 over one meta-cluster. */
+    metaDistill: (cluster: MetaCluster) => Promise<MetaDistillLLMOutput>;
+    publishSkill: (pkg: SkillPackage) => Promise<{
+        envelopeRef: string;
+        anchorTx: string | null;
+    }>;
+    slate: {
+        instanceIds: Set<string>;
+        repos?: Set<string>;
+    };
+    distribution?: string;
+    distillModel?: string;
+    scrubPipeline?: ScrubPipeline;
+    now?: () => Date;
+}
+export interface MetaDistillResult {
+    published: Array<{
+        metaClusterId: string;
+        skillKind: 'cross-instance';
+        envelopeRef: string;
+        pkg: SkillPackage;
+    }>;
+    rejected: Array<{
+        metaClusterId: string;
+        reason: string;
+    }>;
+    errors: Array<{
+        metaClusterId: string;
+        error: string;
+    }>;
+}
+/**
+ * Stage-2 cross-instance meta-distill (issue #1463). For each same-polarity
+ * meta-cluster: run the meta LLM, keep only the sources it names in `supports`,
+ * require **≥2 distinct supporting instances**, union their layer-1 evidence
+ * CIDs into the provenance, and finish through the SAME `finalizeSkill` gate as
+ * stage-1 — so a meta-skill passes the identical scrub / contamination /
+ * structural checks. Additive and opt-in; never called unless the pipeline
+ * enables it.
+ */
+export declare function metaDistill(clusters: MetaCluster[], deps: MetaDistillDeps): Promise<MetaDistillResult>;