@tuned-tensor/local 0.2.9 → 0.3.0

package/dist/labeling-sanitize.d.ts

@@ -1,31 +0,0 @@
-/**
- * Secret/PII sanitization for labeling rows, ported from tuned-tensor-runs.
- * Even though tt-local runs on hardware the user controls, labeling sends row
- * content to OpenRouter — so secret-like content blocks the row and PII is
- * redacted before anything leaves the machine.
- */
-export type SanitizationStatus = "clean" | "redacted" | "blocked";
-export type SanitizationKind = "api_key" | "bearer_token" | "connection_string" | "credit_card" | "email" | "password" | "phone" | "private_key" | "ssn";
-export interface SanitizationFinding {
-    kind: SanitizationKind;
-    action: "redact" | "block";
-    count: number;
-}
-export interface SanitizedText {
-    text: string;
-    status: SanitizationStatus;
-    findings: SanitizationFinding[];
-}
-export interface SanitizedLabelingRow {
-    input: string;
-    output?: string;
-    sanitizationStatus: SanitizationStatus;
-    sanitizationFindings: SanitizationFinding[];
-    sanitizationError?: string;
-}
-export declare function sanitizeText(text: string): SanitizedText;
-export declare function mergeSanitizationFindings(findings: readonly SanitizationFinding[]): SanitizationFinding[];
-export declare function sanitizeLabelingRow(row: {
-    input: string;
-    output?: string;
-}): SanitizedLabelingRow;

package/dist/labeling-sanitize.js

@@ -1,158 +0,0 @@
-/**
- * Secret/PII sanitization for labeling rows, ported from tuned-tensor-runs.
- * Even though tt-local runs on hardware the user controls, labeling sends row
- * content to OpenRouter — so secret-like content blocks the row and PII is
- * redacted before anything leaves the machine.
- */
-const BLOCK_PATTERNS = [
-    {
-        kind: "private_key",
-        replacement: "[REDACTED_PRIVATE_KEY]",
-        regex: /-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/g,
-    },
-    {
-        kind: "connection_string",
-        replacement: "[REDACTED_CONNECTION_STRING]",
-        regex: /\b(?:postgres(?:ql)?|mysql|mongodb(?:\+srv)?|redis):\/\/[^\s:/@]+:[^\s@]+@[^\s]+/gi,
-    },
-    {
-        kind: "bearer_token",
-        replacement: "Bearer [REDACTED_TOKEN]",
-        regex: /\bBearer\s+[A-Za-z0-9._~+/=-]{20,}\b/g,
-    },
-    {
-        kind: "api_key",
-        replacement: "[REDACTED_API_KEY]",
-        regex: /\b(?:sk-[A-Za-z0-9]{20,}|(?:tt|ghp|gho|github_pat|xox[baprs])_[A-Za-z0-9_=-]{20,}|(?:AKIA|ASIA)[A-Z0-9]{16}|AIza[0-9A-Za-z_-]{20,})\b/g,
-    },
-    {
-        kind: "password",
-        replacement: "$1 [REDACTED_SECRET]",
-        regex: /\b((?:password|passwd|pwd|api[_-]?key|secret|token|access[_-]?token|refresh[_-]?token|client[_-]?secret)\s*[:=])\s*["']?[^"'\s,;]{8,}/gi,
-    },
-];
-const REDACT_PATTERNS = [
-    {
-        kind: "email",
-        replacement: "[REDACTED_EMAIL]",
-        regex: /\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi,
-    },
-    {
-        kind: "ssn",
-        replacement: "[REDACTED_SSN]",
-        regex: /\b\d{3}-\d{2}-\d{4}\b/g,
-    },
-    {
-        kind: "phone",
-        replacement: "[REDACTED_PHONE]",
-        regex: /\b(?:\+?1[\s.-]?)?(?:\(?\d{3}\)?[\s.-]?)\d{3}[\s.-]?\d{4}\b/g,
-    },
-];
-function addFinding(findings, kind, action, count) {
-    if (count <= 0)
-        return;
-    const key = `${kind}:${action}`;
-    const existing = findings.get(key);
-    findings.set(key, {
-        kind,
-        action,
-        count: (existing?.count ?? 0) + count,
-    });
-}
-function replacePattern(text, pattern, action, findings) {
-    let count = 0;
-    const next = text.replace(pattern.regex, (...args) => {
-        count += 1;
-        if (pattern.replacement.includes("$1") && typeof args[1] === "string") {
-            return pattern.replacement.replace("$1", args[1]);
-        }
-        return pattern.replacement;
-    });
-    addFinding(findings, pattern.kind, action, count);
-    return next;
-}
-function digitsOnly(value) {
-    return value.replace(/\D/g, "");
-}
-function passesLuhn(value) {
-    let sum = 0;
-    let doubleDigit = false;
-    for (let i = value.length - 1; i >= 0; i -= 1) {
-        let digit = Number(value[i]);
-        if (Number.isNaN(digit))
-            return false;
-        if (doubleDigit) {
-            digit *= 2;
-            if (digit > 9)
-                digit -= 9;
-        }
-        sum += digit;
-        doubleDigit = !doubleDigit;
-    }
-    return sum > 0 && sum % 10 === 0;
-}
-function redactCreditCards(text, findings) {
-    let count = 0;
-    const next = text.replace(/\b(?:\d[ -]*?){13,19}\b/g, (match) => {
-        const digits = digitsOnly(match);
-        if (digits.length < 13 || digits.length > 19 || !passesLuhn(digits)) {
-            return match;
-        }
-        count += 1;
-        return "[REDACTED_CARD]";
-    });
-    addFinding(findings, "credit_card", "redact", count);
-    return next;
-}
-export function sanitizeText(text) {
-    const findings = new Map();
-    let sanitized = text;
-    for (const pattern of BLOCK_PATTERNS) {
-        sanitized = replacePattern(sanitized, pattern, "block", findings);
-    }
-    sanitized = redactCreditCards(sanitized, findings);
-    for (const pattern of REDACT_PATTERNS) {
-        sanitized = replacePattern(sanitized, pattern, "redact", findings);
-    }
-    const allFindings = Array.from(findings.values());
-    const hasBlock = allFindings.some((finding) => finding.action === "block");
-    return {
-        text: sanitized,
-        status: hasBlock
-            ? "blocked"
-            : allFindings.length > 0
-                ? "redacted"
-                : "clean",
-        findings: allFindings,
-    };
-}
-export function mergeSanitizationFindings(findings) {
-    const merged = new Map();
-    for (const finding of findings) {
-        addFinding(merged, finding.kind, finding.action, finding.count);
-    }
-    return Array.from(merged.values());
-}
-export function sanitizeLabelingRow(row) {
-    const input = sanitizeText(row.input);
-    const output = row.output === undefined ? undefined : sanitizeText(row.output);
-    const findings = mergeSanitizationFindings([
-        ...input.findings,
-        ...(output?.findings ?? []),
-    ]);
-    const status = input.status === "blocked" || output?.status === "blocked"
-        ? "blocked"
-        : input.status === "redacted" || output?.status === "redacted"
-            ? "redacted"
-            : "clean";
-    return {
-        input: input.text,
-        ...(output !== undefined ? { output: output.text } : {}),
-        sanitizationStatus: status,
-        sanitizationFindings: findings,
-        ...(status === "blocked"
-            ? { sanitizationError: "Sensitive secret-like content was blocked" }
-            : {}),
-    };
-}
-//# sourceMappingURL=labeling-sanitize.js.map

package/dist/labeling-sanitize.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"labeling-sanitize.js","sourceRoot":"","sources":["../src/labeling-sanitize.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAmCH,MAAM,cAAc,GAIf;IACH;QACE,IAAI,EAAE,aAAa;QACnB,WAAW,EAAE,wBAAwB;QACrC,KAAK,EACH,6EAA6E;KAChF;IACD;QACE,IAAI,EAAE,mBAAmB;QACzB,WAAW,EAAE,8BAA8B;QAC3C,KAAK,EACH,oFAAoF;KACvF;IACD;QACE,IAAI,EAAE,cAAc;QACpB,WAAW,EAAE,yBAAyB;QACtC,KAAK,EAAE,uCAAuC;KAC/C;IACD;QACE,IAAI,EAAE,SAAS;QACf,WAAW,EAAE,oBAAoB;QACjC,KAAK,EACH,wIAAwI;KAC3I;IACD;QACE,IAAI,EAAE,UAAU;QAChB,WAAW,EAAE,sBAAsB;QACnC,KAAK,EACH,yIAAyI;KAC5I;CACF,CAAC;AAEF,MAAM,eAAe,GAIhB;IACH;QACE,IAAI,EAAE,OAAO;QACb,WAAW,EAAE,kBAAkB;QAC/B,KAAK,EAAE,6CAA6C;KACrD;IACD;QACE,IAAI,EAAE,KAAK;QACX,WAAW,EAAE,gBAAgB;QAC7B,KAAK,EAAE,wBAAwB;KAChC;IACD;QACE,IAAI,EAAE,OAAO;QACb,WAAW,EAAE,kBAAkB;QAC/B,KAAK,EAAE,8DAA8D;KACtE;CACF,CAAC;AAEF,SAAS,UAAU,CACjB,QAA0C,EAC1C,IAAsB,EACtB,MAA0B,EAC1B,KAAa;IAEb,IAAI,KAAK,IAAI,CAAC;QAAE,OAAO;IACvB,MAAM,GAAG,GAAG,GAAG,IAAI,IAAI,MAAM,EAAE,CAAC;IAChC,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACnC,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE;QAChB,IAAI;QACJ,MAAM;QACN,KAAK,EAAE,CAAC,QAAQ,EAAE,KAAK,IAAI,CAAC,CAAC,GAAG,KAAK;KACtC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,cAAc,CACrB,IAAY,EACZ,OAAuE,EACvE,MAA0B,EAC1B,QAA0C;IAE1C,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,GAAG,IAAe,EAAE,EAAE;QAC9D,KAAK,IAAI,CAAC,CAAC;QACX,IAAI,OAAO,CAAC,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,OAAO,IAAI,CAAC,CAAC,CAAC,KAAK,QAAQ,EAAE,CAAC;YACtE,OAAO,OAAO,CAAC,WAAW,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QACpD,CAAC;QACD,OAAO,OAAO,CAAC,WAAW,CAAC;IAC7B,CAAC,CAAC,CAAC;IACH,UAAU,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;IAClD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,UAAU,CAAC,KAAa;IAC/B,OAAO,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;AAClC,CAAC;AAED,SAAS,UAAU,CAAC,KAAa;IAC/B,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,WAAW,GAAG,KAAK,CAAC;IACxB,KAAK,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QAC9C,IAAI,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7B,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC;QACtC,IAAI,WAAW,EAAE,CAAC;YAChB,KAAK,IAAI,CAAC,CAAC;YACX,IAAI,KAAK,GAAG,CAAC;gBAAE,KAAK,IAAI,CAAC,CAAC;QAC5B,CAAC;QACD,GAAG,IAAI,KAAK,CAAC;QACb,WAAW,GAAG,CAAC,WAAW,CAAC;IAC7B,CAAC;IACD,OAAO,GAAG,GAAG,CAAC,IAAI,GAAG,GAAG,EAAE,KAAK,CAAC,CAAC;AACnC,CAAC;AAED,SAAS,iBAAiB,CACxB,IAAY,EACZ,QAA0C;IAE1C,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,0BAA0B,EAAE,CAAC,KAAK,EAAE,EAAE;QAC9D,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;QACjC,IAAI,MAAM,CAAC,MAAM,GAAG,EAAE,IAAI,MAAM,CAAC,MAAM,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YACpE,OAAO,KAAK,CAAC;QACf,CAAC;QACD,KAAK,IAAI,CAAC,CAAC;QACX,OAAO,iBAAiB,CAAC;IAC3B,CAAC,CAAC,CAAC;IACH,UAAU,CAAC,QAAQ,EAAE,aAAa,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;IACrD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,IAAY;IACvC,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA+B,CAAC;IACxD,IAAI,SAAS,GAAG,IAAI,CAAC;IAErB,KAAK,MAAM,OAAO,IAAI,cAAc,EAAE,CAAC;QACrC,SAAS,GAAG,cAAc,CAAC,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;IACpE,CAAC;IAED,SAAS,GAAG,iBAAiB,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;IAEnD,KAAK,MAAM,OAAO,IAAI,eAAe,EAAE,CAAC;QACtC,SAAS,GAAG,cAAc,CAAC,SAAS,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;IACrE,CAAC;IAED,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;IAClD,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,KAAK,OAAO,CAAC,CAAC;IAC3E,OAAO;QACL,IAAI,EAAE,SAAS;QACf,MAAM,EAAE,QAAQ;YACd,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC;gBACtB,CAAC,CAAC,UAAU;gBACZ,CAAC,CAAC,OAAO;QACb,QAAQ,EAAE,WAAW;KACtB,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,yBAAyB,CACvC,QAAwC;IAExC,MAAM,MAAM,GAAG,IAAI,GAAG,EAA+B,CAAC;IACtD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,UAAU,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IAClE,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;AACrC,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,GAGnC;IACC,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IACtC,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAC/E,MAAM,QAAQ,GAAG,yBAAyB,CAAC;QACzC,GAAG,KAAK,CAAC,QAAQ;QACjB,GAAG,CAAC,MAAM,EAAE,QAAQ,IAAI,EAAE,CAAC;KAC5B,CAAC,CAAC;IACH,MAAM,MAAM,GACV,KAAK,CAAC,MAAM,KAAK,SAAS,IAAI,MAAM,EAAE,MAAM,KAAK,SAAS;QACxD,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,KAAK,CAAC,MAAM,KAAK,UAAU,IAAI,MAAM,EAAE,MAAM,KAAK,UAAU;YAC5D,CAAC,CAAC,UAAU;YACZ,CAAC,CAAC,OAAO,CAAC;IAEhB,OAAO;QACL,KAAK,EAAE,KAAK,CAAC,IAAI;QACjB,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACxD,kBAAkB,EAAE,MAAM;QAC1B,oBAAoB,EAAE,QAAQ;QAC9B,GAAG,CAAC,MAAM,KAAK,SAAS;YACtB,CAAC,CAAC,EAAE,iBAAiB,EAAE,2CAA2C,EAAE;YACpE,CAAC,CAAC,EAAE,CAAC;KACR,CAAC;AACJ,CAAC"}

package/dist/labeling.d.ts

@@ -1,155 +0,0 @@
-/**
- * Local teacher-labeling job, ported from the tuned-tensor-runs labeling
- * workflow. Reads an unlabeled JSONL or CSV source, sanitizes each row, sends
- * pending rows to an OpenRouter teacher model under the spec's system message
- * (the same message the fine-tuned model will see at inference), and writes a
- * labeled {"input","output"} JSONL plus a job report under the artifact root.
- *
- * Unlike the eval judge there is no JSON output schema (the teacher's output
- * IS the training label, free text) and no heuristic fallback — a failed call
- * must surface as a failed row, never a fabricated label.
- */
-import type { LocalRunnerConfig } from "./contracts.js";
-import { type SanitizationFinding, type SanitizationStatus } from "./labeling-sanitize.js";
-import type { LocalRunReporter } from "./run-reporter.js";
-/** Per-row input cap; larger documents should be chunked upstream. */
-export declare const MAX_INPUT_CHARS = 32000;
-export declare const MAX_PARSE_ERRORS = 50;
-export interface UnlabeledRow {
-    rowIndex: number;
-    input: string;
-    /** Present when the source row was already labeled (JSONL only). */
-    output?: string;
-}
-export interface FailedUnlabeledRow {
-    rowIndex: number;
-    input: string;
-    error: string;
-}
-export interface ParseResult {
-    rows: UnlabeledRow[];
-    failedRows: FailedUnlabeledRow[];
-    errors: string[];
-}
-/**
- * Parse a JSONL source of {"input": string, "output"?: string} rows. Rows
- * that already carry an "output" skip the teacher and land directly as
- * labeled. Collects errors instead of throwing; callers reject the job when
- * `errors` is non-empty.
- */
-export declare function parseUnlabeledJsonl(text: string, maxRows: number): ParseResult;
-/**
- * Minimal RFC 4180 CSV reader (quoted fields, escaped quotes, CR/LF inside
- * quotes). tt-local deliberately avoids a CSV dependency; this covers the
- * upload formats the hosted labeling dialog accepts.
- */
-export declare function parseCsvRecords(text: string): {
-    fields: string[];
-    records: string[][];
-    errors: string[];
-};
-/**
- * Parse a CSV source, taking `inputColumn` as the input text of each row.
- * The header row is required.
- */
-export declare function parseUnlabeledCsv(text: string, inputColumn: string, maxRows: number): ParseResult;
-/**
- * Defensive removal of `<think>` reasoning blocks so the training label is
- * only the teacher's final answer. Truncated output can leave a dangling
- * close tag; keep only what follows the last close tag in that case.
- */
-export declare function stripModelThinking(raw: string): string;
-/**
- * Map over items with bounded concurrency and optional minimum spacing
- * between call starts (global across workers).
- */
-export declare function mapWithPacing<T, R>(items: readonly T[], pacing: {
-    concurrency: number;
-    minIntervalMs: number;
-}, mapper: (item: T, index: number) => Promise<R>): Promise<R[]>;
-export interface TeacherOptions {
-    model: string;
-    apiKeyEnv: string;
-    appName?: string;
-    siteUrl?: string;
-    maxTokens: number;
-    temperature: number;
-    maxAttempts: number;
-    timeoutMs: number;
-}
-export interface TeacherLabelResult {
-    output: string;
-    promptTokens: number;
-    completionTokens: number;
-    latencyMs: number;
-}
-export declare function resolveTeacherOptions(config: LocalRunnerConfig, overrides?: {
-    model?: string;
-}): TeacherOptions;
-export declare function labelRow(args: {
-    systemMessage: string;
-    input: string;
-    teacher: TeacherOptions;
-}): Promise<TeacherLabelResult>;
-export type LabelingRowStatus = "labeled" | "failed" | "pending";
-export interface LabelingRowResult {
-    row_index: number;
-    input: string;
-    output?: string;
-    status: LabelingRowStatus;
-    label_source?: "teacher" | "upload";
-    error?: string;
-    sanitization_status: SanitizationStatus;
-    sanitization_findings: SanitizationFinding[];
-    prompt_tokens: number;
-    completion_tokens: number;
-}
-export interface LocalLabelingReport {
-    job_id: string;
-    status: "completed" | "failed" | "dry_run";
-    source_path: string;
-    source_format: "jsonl" | "csv";
-    input_column?: string;
-    teacher_model_id: string;
-    row_count: number;
-    labeled_count: number;
-    prelabeled_count: number;
-    failed_count: number;
-    redacted_count: number;
-    blocked_count: number;
-    pending_count: number;
-    prompt_tokens_total: number;
-    completion_tokens_total: number;
-    error?: string;
-    artifact_uris: {
-        labeled: string;
-        rows: string;
-        report: string;
-    };
-    created_at: string;
-    completed_at: string;
-}
-export interface LocalLabelingResult {
-    report: LocalLabelingReport;
-    reportPath: string;
-    artifactDir: string;
-    labeledPath: string;
-    rowsPath: string;
-}
-export interface RunLocalLabelingJobArgs {
-    sourcePath: string;
-    /** Inferred from the file extension when omitted. */
-    format?: "jsonl" | "csv";
-    /** Required for CSV sources. */
-    inputColumn?: string;
-    systemMessage: string;
-    config: LocalRunnerConfig;
-    /** Optional extra copy of the labeled JSONL (artifact copy is always written). */
-    outputPath?: string;
-    /** Teacher model override; falls back to labeling.model then llm.model. */
-    model?: string;
-    /** Parse and sanitize only; no teacher calls, no labeled.jsonl. */
-    dryRun?: boolean;
-    reporter?: LocalRunReporter;
-}
-export declare function runLocalLabelingJob(args: RunLocalLabelingJobArgs): Promise<LocalLabelingResult>;