osborn 0.9.251 → 0.9.252
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/content/compaction-lens-worker.d.ts +7 -2
- package/dist/content/compaction-lens-worker.js +60 -7
- package/dist/content/compaction-lens.js +13 -2
- package/dist/content/content-brief.d.ts +90 -0
- package/dist/content/content-brief.js +165 -0
- package/dist/content/content-checks.d.ts +88 -0
- package/dist/content/content-checks.js +249 -0
- package/dist/content/content-ingest.d.ts +51 -0
- package/dist/content/content-ingest.js +107 -0
- package/dist/content/content-llm.d.ts +134 -0
- package/dist/content/content-llm.js +412 -0
- package/dist/content/content-manifest.d.ts +115 -0
- package/dist/content/content-manifest.js +59 -0
- package/dist/content/content-query-gate.d.ts +22 -0
- package/dist/content/content-query-gate.js +78 -0
- package/dist/content/content-redact.d.ts +22 -0
- package/dist/content/content-redact.js +49 -0
- package/dist/content/content-research.d.ts +90 -0
- package/dist/content/content-research.js +256 -0
- package/dist/content/content-run.d.ts +85 -0
- package/dist/content/content-run.js +321 -0
- package/dist/content/content-script-rules.d.ts +74 -0
- package/dist/content/content-script-rules.js +134 -0
- package/dist/content/content-script.d.ts +61 -0
- package/dist/content/content-script.js +107 -0
- package/dist/content/lens-ingest.d.ts +7 -0
- package/dist/content/lens-ingest.js +21 -4
- package/dist/content/transcript-sanitizer.js +6 -0
- package/package.json +1 -1
|
@@ -0,0 +1,249 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* content-checks.ts — Stage A gates. A HARD flag ⇒ the piece is "blocked":
|
|
3
|
+
* never ingested, never auto-fixed (content-run records the flags).
|
|
4
|
+
* HARD always: dev-voice pre-pass, truth-check, length below the tier min or
|
|
5
|
+
* more than +15% over the tier max, an empty script.
|
|
6
|
+
* ADVISORY by default (warnings → the draft's quality_flags; still ingested):
|
|
7
|
+
* the structure gate (stake / "no visible proof", causes, terms/min, spoken
|
|
8
|
+
* versions/codes), up to +15% over the tier max, and the audience check.
|
|
9
|
+
* OSBORN_CONTENT_STRICT=1 makes every advisory warning a hard flag again.
|
|
10
|
+
*
|
|
11
|
+
* Order (cheapest first; a failing stage skips the paid ones after it):
|
|
12
|
+
* 1. length deterministic: estimated seconds within the tier bounds.
|
|
13
|
+
* 2. dev-voice deterministic pre-pass (rules §7): every dev line must be a
|
|
14
|
+
* substring of ONE user row of the period. Ellipsis cuts ("…" / "...")
|
|
15
|
+
* are allowed only if every piece is ≥12 chars, the pieces occur in
|
|
16
|
+
* order in that row, and they cover ≥80% of the row span they were
|
|
17
|
+
* cut from (so a cut can't flip the meaning). Records the row id.
|
|
18
|
+
* Also caps dev lines per tier (a highlight is not a quote chain).
|
|
19
|
+
* 3. truth-check LLM, PINNED: deepseek/deepseek-chat on deepinfra/fp4 with
|
|
20
|
+
* allow_fallbacks:false (as videos-test-3/vt3-render.ts); the served
|
|
21
|
+
* provider must be DeepInfra. Cached by sha256(draft+sources+prompt+model+provider).
|
|
22
|
+
* 4. audience LLM (same pin): rules §4 — every quote/term familiar to the viewer or set up nearby.
|
|
23
|
+
*/
|
|
24
|
+
import { createHash } from 'node:crypto';
|
|
25
|
+
import { mkdirSync, readFileSync } from 'node:fs';
|
|
26
|
+
import { join } from 'node:path';
|
|
27
|
+
import { CHECK_MODEL, CHECK_PROVIDER, CHECK_PROVIDER_NAME, isContentStrict, llmJson } from './content-llm.js';
|
|
28
|
+
import { writeAtomic } from './content-manifest.js';
|
|
29
|
+
import { AUDIENCE_RULES, TIER_TEMPLATES, scriptLimits } from './content-script-rules.js';
|
|
30
|
+
import { lengthCheck } from './content-script.js';
|
|
31
|
+
export const CHECK_CACHE_DIR = '.content-check-cache';
|
|
32
|
+
export const MIN_PIECE_CHARS = 12;
|
|
33
|
+
export const MIN_CUT_COVERAGE = 0.8;
|
|
34
|
+
/** Non-strict: a script may run up to this fraction over the tier max (flagged, not blocked). */
|
|
35
|
+
export const OVER_LENGTH_TOLERANCE = 0.15;
|
|
36
|
+
export const NO_PROOF_FLAG = 'no visible proof: brief has no before -> after stake quantity (rule 1: stake and proof both visible)';
|
|
37
|
+
/** Whitespace + typographic quote/dash normalisation only (never wording). */
|
|
38
|
+
export const normText = (s) => String(s ?? '')
|
|
39
|
+
.replace(/[‘’‛′]/g, "'")
|
|
40
|
+
.replace(/[“”‟″]/g, '"')
|
|
41
|
+
.replace(/[–—]/g, '-')
|
|
42
|
+
.replace(/\s+/g, ' ')
|
|
43
|
+
.trim();
|
|
44
|
+
const ELLIPSIS = /\s*(?:…|\.\.\.)\s*/;
|
|
45
|
+
const stripWrapQuotes = (s) => s.replace(/^["'\s]+|["'\s]+$/g, '');
|
|
46
|
+
/** Pieces of a dev line split at ellipsis cuts (leading/trailing cuts allowed). */
|
|
47
|
+
export const devPieces = (text) => normText(text).split(ELLIPSIS).map(stripWrapQuotes).filter(Boolean);
|
|
48
|
+
/** Match pieces in order inside `row`. Returns the covered span ratio, or null when not all found. */
|
|
49
|
+
export function matchInRow(row, pieces) {
|
|
50
|
+
const hay = normText(row);
|
|
51
|
+
let at = 0;
|
|
52
|
+
let first = -1;
|
|
53
|
+
let end = 0;
|
|
54
|
+
for (const p of pieces) {
|
|
55
|
+
const k = hay.indexOf(p, at);
|
|
56
|
+
if (k < 0)
|
|
57
|
+
return null;
|
|
58
|
+
if (first < 0)
|
|
59
|
+
first = k;
|
|
60
|
+
at = k + p.length;
|
|
61
|
+
end = at;
|
|
62
|
+
}
|
|
63
|
+
const kept = pieces.reduce((n, p) => n + p.length, 0);
|
|
64
|
+
return end > first ? kept / (end - first) : 1;
|
|
65
|
+
}
|
|
66
|
+
/** Deterministic dev-voice pre-pass. `rows` = the period's records (only user rows are eligible). */
|
|
67
|
+
export function devVoicePrepass(lines, rows, tier) {
|
|
68
|
+
const users = rows.filter(r => r.speaker === 'user');
|
|
69
|
+
const res = { ok: true, flags: [], rows: [] };
|
|
70
|
+
let devCount = 0;
|
|
71
|
+
lines.forEach((l, i) => {
|
|
72
|
+
if (l.speaker !== 'dev')
|
|
73
|
+
return;
|
|
74
|
+
devCount++;
|
|
75
|
+
const n = i + 1;
|
|
76
|
+
const pieces = devPieces(l.text);
|
|
77
|
+
if (!pieces.length)
|
|
78
|
+
return void res.flags.push(`dev line ${n}: empty`);
|
|
79
|
+
const short = pieces.find(p => p.length < MIN_PIECE_CHARS);
|
|
80
|
+
if (short)
|
|
81
|
+
return void res.flags.push(`dev line ${n}: piece "${short}" is under ${MIN_PIECE_CHARS} chars`);
|
|
82
|
+
// The cited row first, then every other user row.
|
|
83
|
+
const order = [...users.filter(r => r.id === l.row), ...users.filter(r => r.id !== l.row)];
|
|
84
|
+
let best = null;
|
|
85
|
+
for (const r of order) {
|
|
86
|
+
const cov = matchInRow(r.text, pieces);
|
|
87
|
+
if (cov === null)
|
|
88
|
+
continue;
|
|
89
|
+
if (pieces.length === 1 || cov >= MIN_CUT_COVERAGE) {
|
|
90
|
+
best = { row: r.id, cov };
|
|
91
|
+
break;
|
|
92
|
+
}
|
|
93
|
+
if (!best || cov > best.cov)
|
|
94
|
+
best = { row: r.id, cov };
|
|
95
|
+
}
|
|
96
|
+
if (!best)
|
|
97
|
+
return void res.flags.push(`dev line ${n}: not verbatim in any user row of the period`);
|
|
98
|
+
if (pieces.length > 1 && best.cov < MIN_CUT_COVERAGE) {
|
|
99
|
+
return void res.flags.push(`dev line ${n}: ellipsis cuts keep ${(best.cov * 100).toFixed(0)}% of row #${best.row} span (< ${MIN_CUT_COVERAGE * 100}%)`);
|
|
100
|
+
}
|
|
101
|
+
res.rows.push({ line: n, row: best.row, match: pieces.length > 1 ? `ellipsis ${(best.cov * 100).toFixed(0)}%` : 'substring' });
|
|
102
|
+
});
|
|
103
|
+
if (tier && devCount > TIER_TEMPLATES[tier].devLines.max)
|
|
104
|
+
res.flags.push(`${devCount} dev lines > ${TIER_TEMPLATES[tier].devLines.max} allowed for ${tier} (not a quote chain)`);
|
|
105
|
+
res.ok = res.flags.length === 0;
|
|
106
|
+
return res;
|
|
107
|
+
}
|
|
108
|
+
const TAG = {
|
|
109
|
+
dev: '[DEVELOPER, verbatim user line; … marks elided words]',
|
|
110
|
+
agent: '[AGENT]',
|
|
111
|
+
narrator: '[NARRATOR]',
|
|
112
|
+
};
|
|
113
|
+
export const draftText = (s) => `ON-SCREEN TITLE: ${s.title}\nHOOK: ${s.hook}\n\nNARRATION (one line per caption):\n` + s.lines.map((l, i) => `${i + 1}. ${TAG[l.speaker]} ${l.text}`).join('\n');
|
|
114
|
+
/** pipeline-e2e LEG 5 / vt3-render.ts truth-check prompt, unchanged. */
|
|
115
|
+
export const TRUTH_SYSTEM = 'You are a strict fact-checking editor. You are given a DRAFT post and the SOURCE records (the ONLY ' +
|
|
116
|
+
'ground truth). Identify every factual claim, event, metric, or specific detail in the draft that is ' +
|
|
117
|
+
'NOT directly supported by the source records. Treat invented stories, numbers, or named mechanisms as ' +
|
|
118
|
+
'unsupported. Do NOT rewrite or soften the post. Return ONLY a JSON array of strings — each string one ' +
|
|
119
|
+
'specific ungrounded claim (quote or tight paraphrase from the draft). If every claim is grounded, return [].';
|
|
120
|
+
export const truthUser = (sources, draft) => `SOURCE RECORDS (ground truth, speaker-tagged):\n${sources}\n\nDRAFT POST:\n${draft}\n\nList the ungrounded claims as a JSON array of strings.`;
|
|
121
|
+
export const AUDIENCE_SYSTEM = 'You are an editor checking a short developer video script for ONE target viewer. Reply with ONE JSON object only.';
|
|
122
|
+
export const audienceUser = (brief, draft) => `${AUDIENCE_RULES}\n\nTARGET VIEWER: ${brief.viewer}\nTHEY ARE LIVING THROUGH: ${brief.situation}\n\nSCRIPT:\n${draft}\n\n` +
|
|
123
|
+
'List each quote or term that lands with no context for this viewer. Output ONLY: {"flags": [{"item": quote or term, "why": one line}]} — [] when every one passes.';
|
|
124
|
+
/** Flags from any reply shape: ["..."], {flags:[...]}, [{claim|item|text}], … */
|
|
125
|
+
export function flagsFrom(json) {
|
|
126
|
+
let arr = json;
|
|
127
|
+
if (json && typeof json === 'object' && !Array.isArray(json)) {
|
|
128
|
+
const o = json;
|
|
129
|
+
arr = [o.flags, o.unsupported, o.issues, o.claims, o.unfamiliar].find(Array.isArray) ?? [];
|
|
130
|
+
}
|
|
131
|
+
if (!Array.isArray(arr))
|
|
132
|
+
return [];
|
|
133
|
+
return arr
|
|
134
|
+
.map(x => (typeof x === 'string' ? x : x && typeof x === 'object' ? [x.claim ?? x.item ?? x.text ?? x.term ?? '', x.why ?? x.reason ?? ''].filter(Boolean).join(' — ') : ''))
|
|
135
|
+
.map(s => String(s).replace(/\s+/g, ' ').trim())
|
|
136
|
+
.filter(Boolean)
|
|
137
|
+
.slice(0, 20);
|
|
138
|
+
}
|
|
139
|
+
/** Spoken software versions / error codes (WHAT-WORKS rule 4). Starting patterns; dev lines are exempt (verbatim). */
|
|
140
|
+
export const VERSION_CODE_PATTERNS = [
|
|
141
|
+
/\b\d+\.\d+\.\d+\b/, // 0.9.73
|
|
142
|
+
/\bv\d+(?:\.\d+)+\b/i, // v2.1
|
|
143
|
+
/\bversion\s+\d+(?:\.\d+)*\b/i, // version 186
|
|
144
|
+
/\b(?:HTTP|status|error|exit|code)\s*(?:code\s*)?\d{3,5}\b/i, // HTTP 502, error 1006, exit code 137
|
|
145
|
+
/\b\d{3,5}\s+(?:error|status code)\b/i, // 404 error
|
|
146
|
+
/\b(?:ws|websocket)\s+\d{4}\b/i, // WS 1006
|
|
147
|
+
/\bE[A-Z]{4,}\b/, // ENOENT, EADDRINUSE, ECONNRESET
|
|
148
|
+
];
|
|
149
|
+
/** WHAT-WORKS deterministic gate, run on the script BEFORE any LLM judge. Thresholds: scriptLimits() (env-tunable). */
|
|
150
|
+
export function structureCheck(s, brief, limits = scriptLimits()) {
|
|
151
|
+
const flags = [];
|
|
152
|
+
if (!brief.stake)
|
|
153
|
+
flags.push(NO_PROOF_FLAG);
|
|
154
|
+
const seconds = lengthCheck(s).seconds;
|
|
155
|
+
const causes = s.causes.length;
|
|
156
|
+
const maxC = limits.maxCauses[s.tier];
|
|
157
|
+
if (causes === 0)
|
|
158
|
+
flags.push('script declares no cause');
|
|
159
|
+
else {
|
|
160
|
+
if (causes > maxC)
|
|
161
|
+
flags.push(`${causes} causes > ${maxC} allowed for ${s.tier}`);
|
|
162
|
+
if (seconds / causes < limits.secondsPerCause)
|
|
163
|
+
flags.push(`${(seconds / causes).toFixed(0)}s per cause < ${limits.secondsPerCause}s (${causes} causes in ~${seconds.toFixed(0)}s)`);
|
|
164
|
+
}
|
|
165
|
+
const termsPerMin = seconds > 0 ? Number((s.terms.length / (seconds / 60)).toFixed(2)) : 0;
|
|
166
|
+
if (termsPerMin > limits.maxTermsPerMin[s.tier])
|
|
167
|
+
flags.push(`${termsPerMin} load-bearing terms/min > ${limits.maxTermsPerMin[s.tier]} (${s.terms.length} terms)`);
|
|
168
|
+
const spoken = [{ n: 0, text: s.title }, { n: 0, text: s.hook }, ...s.lines.map((l, i) => ({ n: i + 1, text: l.speaker === 'dev' ? '' : l.text }))];
|
|
169
|
+
for (const { n, text } of spoken) {
|
|
170
|
+
const re = VERSION_CODE_PATTERNS.find(p => p.test(text));
|
|
171
|
+
if (re)
|
|
172
|
+
flags.push(`${n ? `line ${n}` : 'title/hook'}: spoken version or error code ("${text.match(re)[0]}")`);
|
|
173
|
+
}
|
|
174
|
+
return { ok: flags.length === 0, flags, causes, termsPerMin };
|
|
175
|
+
}
|
|
176
|
+
function cachePath(projectDir, key) {
|
|
177
|
+
return projectDir ? join(projectDir, CHECK_CACHE_DIR, `${key}.json`) : null;
|
|
178
|
+
}
|
|
179
|
+
/** Pinned LLM judge with a content-addressed cache. Throws CapError / Error. */
|
|
180
|
+
async function pinnedJudge(kind, system, user, ctx) {
|
|
181
|
+
const key = createHash('sha256').update(JSON.stringify({ kind, system, user, model: CHECK_MODEL, provider: CHECK_PROVIDER })).digest('hex');
|
|
182
|
+
const p = cachePath(ctx.projectDir, key);
|
|
183
|
+
if (p) {
|
|
184
|
+
try {
|
|
185
|
+
const c = JSON.parse(readFileSync(p, 'utf-8'));
|
|
186
|
+
if (c?.key === key && Array.isArray(c.flags))
|
|
187
|
+
return { flags: c.flags, provider: String(c.provider), cacheKey: key, cached: true };
|
|
188
|
+
}
|
|
189
|
+
catch {
|
|
190
|
+
/* miss */
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
const r = await llmJson(ctx.llm, {
|
|
194
|
+
what: kind === 'truth' ? 'truth-check' : 'audience-check', system, user, maxOut: 1200, temperature: 0,
|
|
195
|
+
model: CHECK_MODEL, pin: { endpoint: CHECK_PROVIDER, name: CHECK_PROVIDER_NAME },
|
|
196
|
+
});
|
|
197
|
+
if (r.json === null)
|
|
198
|
+
throw new Error(`${kind}-check: unparseable reply`);
|
|
199
|
+
const flags = flagsFrom(r.json);
|
|
200
|
+
if (p) {
|
|
201
|
+
try {
|
|
202
|
+
mkdirSync(join(ctx.projectDir, CHECK_CACHE_DIR), { recursive: true });
|
|
203
|
+
writeAtomic(p, JSON.stringify({ key, kind, model: CHECK_MODEL, provider: r.provider, flags, at: new Date().toISOString() }, null, 2) + '\n');
|
|
204
|
+
}
|
|
205
|
+
catch {
|
|
206
|
+
/* best-effort */
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
return { flags, provider: r.provider, cacheKey: key, cached: false };
|
|
210
|
+
}
|
|
211
|
+
/**
|
|
212
|
+
* All gates for one script. `flags` (hard) non-empty ⇒ blocked. `warnings` are advisory
|
|
213
|
+
* quality flags (empty under OSBORN_CONTENT_STRICT=1, where they are hard flags instead).
|
|
214
|
+
* Throws CapError / Error (piece → capped / error).
|
|
215
|
+
*/
|
|
216
|
+
export async function runChecks(s, brief, ctx) {
|
|
217
|
+
const strict = isContentStrict();
|
|
218
|
+
const length = lengthCheck(s);
|
|
219
|
+
const prepass = devVoicePrepass(s.lines, ctx.records, s.tier);
|
|
220
|
+
const structure = structureCheck(s, brief);
|
|
221
|
+
const summary = { prepass, truth: null, audience: null, length, structure };
|
|
222
|
+
const flags = [];
|
|
223
|
+
const warnings = [];
|
|
224
|
+
// Advisory unless strict: same text either way, only where it lands differs.
|
|
225
|
+
const soft = (f) => (strict ? flags : warnings).push(f);
|
|
226
|
+
if (!length.ok) {
|
|
227
|
+
const msg = `length ~${length.seconds}s outside ${length.min}-${length.max}s for ${s.tier}`;
|
|
228
|
+
const tolerated = length.seconds > length.max && length.seconds <= length.max * (1 + OVER_LENGTH_TOLERANCE);
|
|
229
|
+
if (tolerated)
|
|
230
|
+
soft(`${msg} (within +${OVER_LENGTH_TOLERANCE * 100}% over-length tolerance)`);
|
|
231
|
+
else
|
|
232
|
+
flags.push(msg);
|
|
233
|
+
}
|
|
234
|
+
flags.push(...prepass.flags);
|
|
235
|
+
structure.flags.forEach(soft);
|
|
236
|
+
if (!s.lines.length)
|
|
237
|
+
flags.push('script has no lines');
|
|
238
|
+
if (flags.length)
|
|
239
|
+
return { summary, flags, warnings };
|
|
240
|
+
const draft = draftText(s);
|
|
241
|
+
const t = await pinnedJudge('truth', TRUTH_SYSTEM, truthUser(ctx.sources, draft), ctx);
|
|
242
|
+
summary.truth = { ok: t.flags.length === 0, flags: t.flags, model: CHECK_MODEL, provider: t.provider, cacheKey: t.cacheKey, cached: t.cached };
|
|
243
|
+
if (t.flags.length)
|
|
244
|
+
return { summary, flags: t.flags.map(f => `truth: ${f}`), warnings };
|
|
245
|
+
const a = await pinnedJudge('audience', AUDIENCE_SYSTEM, audienceUser(brief, draft), ctx);
|
|
246
|
+
summary.audience = { ok: a.flags.length === 0, flags: a.flags, model: CHECK_MODEL, provider: a.provider };
|
|
247
|
+
a.flags.forEach(f => soft(`audience: ${f}`));
|
|
248
|
+
return { summary, flags, warnings };
|
|
249
|
+
}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* content-ingest.ts — sends a CHECKED script draft (no flags) to the cloud
|
|
3
|
+
* library through the same content_ingest RPC as library pages
|
|
4
|
+
* (lens-ingest.ts postContentIngest: sync-token auth, anon key, the
|
|
5
|
+
* Content-Profile: public header, size cap, never throws, never sends status).
|
|
6
|
+
*
|
|
7
|
+
* Mapping (no new content type):
|
|
8
|
+
* type text_post (Stage B later attaches media to the SAME row → short / audio_video)
|
|
9
|
+
* source_kind script_highlight | script_howto
|
|
10
|
+
* content_hash sha256(session, period start row, piece id) — stable per piece, so a
|
|
11
|
+
* re-ingest upserts and a later media attach hits the same row
|
|
12
|
+
* source_anchors project_slug (basename(projectDir), as lens-library), period, brief,
|
|
13
|
+
* research citations, check result, quality_flags (advisory warnings),
|
|
14
|
+
* dev-voice row ids, auto_generated: true
|
|
15
|
+
* Every string in the payload goes through scrubDeep() (content-redact.ts) first.
|
|
16
|
+
* Blocked pieces never reach this module (content-run).
|
|
17
|
+
*/
|
|
18
|
+
import { type IngestOptions, type IngestState } from './lens-ingest.js';
|
|
19
|
+
import type { CheckSummary } from './content-manifest.js';
|
|
20
|
+
import type { Brief } from './content-brief.js';
|
|
21
|
+
import { type Script } from './content-script.js';
|
|
22
|
+
export declare const AUTO_LABEL = "Auto-generated draft";
|
|
23
|
+
export interface ScriptDraft {
|
|
24
|
+
sessionId: string;
|
|
25
|
+
projectDir: string | null;
|
|
26
|
+
period: {
|
|
27
|
+
index: number;
|
|
28
|
+
startRowId: number;
|
|
29
|
+
endRowId: number;
|
|
30
|
+
page: string;
|
|
31
|
+
};
|
|
32
|
+
brief: Brief;
|
|
33
|
+
script: Script;
|
|
34
|
+
checks: CheckSummary;
|
|
35
|
+
/** Advisory quality warnings (not blocking) → source_anchors.quality_flags + "Draft quality notes:" in the body. */
|
|
36
|
+
qualityFlags?: string[];
|
|
37
|
+
citations: {
|
|
38
|
+
url: string;
|
|
39
|
+
title: string;
|
|
40
|
+
source: string;
|
|
41
|
+
}[];
|
|
42
|
+
model: string;
|
|
43
|
+
}
|
|
44
|
+
export declare const pieceContentHash: (sessionId: string, startRowId: number, pieceId: string) => string;
|
|
45
|
+
export declare const projectSlugFor: (projectDir: string | null) => string | null;
|
|
46
|
+
/** The draft as markdown (the `body` field), labelled as auto-generated. */
|
|
47
|
+
export declare function scriptMarkdown(d: ScriptDraft): string;
|
|
48
|
+
export declare function buildScriptPayload(d: ScriptDraft, scrubDeep?: <T>(v: T) => T): Record<string, unknown>;
|
|
49
|
+
export declare const isIngestConfigured: () => boolean;
|
|
50
|
+
/** Ingest one checked script draft. Never throws. */
|
|
51
|
+
export declare function ingestScript(d: ScriptDraft, scrubDeep: <T>(v: T) => T, o?: IngestOptions): Promise<IngestState>;
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* content-ingest.ts — sends a CHECKED script draft (no flags) to the cloud
|
|
3
|
+
* library through the same content_ingest RPC as library pages
|
|
4
|
+
* (lens-ingest.ts postContentIngest: sync-token auth, anon key, the
|
|
5
|
+
* Content-Profile: public header, size cap, never throws, never sends status).
|
|
6
|
+
*
|
|
7
|
+
* Mapping (no new content type):
|
|
8
|
+
* type text_post (Stage B later attaches media to the SAME row → short / audio_video)
|
|
9
|
+
* source_kind script_highlight | script_howto
|
|
10
|
+
* content_hash sha256(session, period start row, piece id) — stable per piece, so a
|
|
11
|
+
* re-ingest upserts and a later media attach hits the same row
|
|
12
|
+
* source_anchors project_slug (basename(projectDir), as lens-library), period, brief,
|
|
13
|
+
* research citations, check result, quality_flags (advisory warnings),
|
|
14
|
+
* dev-voice row ids, auto_generated: true
|
|
15
|
+
* Every string in the payload goes through scrubDeep() (content-redact.ts) first.
|
|
16
|
+
* Blocked pieces never reach this module (content-run).
|
|
17
|
+
*/
|
|
18
|
+
import { createHash } from 'node:crypto';
|
|
19
|
+
import { basename } from 'node:path';
|
|
20
|
+
import { postContentIngest, resolveIngestConfig } from './lens-ingest.js';
|
|
21
|
+
import { CHECK_MODEL, CHECK_PROVIDER } from './content-llm.js';
|
|
22
|
+
import { FORMAT_CATALOG, TIER_TEMPLATES } from './content-script-rules.js';
|
|
23
|
+
import { scriptTranscript } from './content-script.js';
|
|
24
|
+
export const AUTO_LABEL = 'Auto-generated draft';
|
|
25
|
+
const SESSION_ID_RE = /^[a-zA-Z0-9._-]{1,128}$/;
|
|
26
|
+
export const pieceContentHash = (sessionId, startRowId, pieceId) => createHash('sha256').update(`content-script\n${sessionId}\n${startRowId}\n${pieceId}`).digest('hex');
|
|
27
|
+
export const projectSlugFor = (projectDir) => (projectDir ? basename(projectDir) : null);
|
|
28
|
+
const LABEL = { narrator: 'NARRATOR', dev: 'DEVELOPER (verbatim)', agent: 'AGENT' };
|
|
29
|
+
/** The draft as markdown (the `body` field), labelled as auto-generated. */
|
|
30
|
+
export function scriptMarkdown(d) {
|
|
31
|
+
const b = d.brief;
|
|
32
|
+
const s = d.script;
|
|
33
|
+
const fmt = FORMAT_CATALOG.find(f => f.id === b.format);
|
|
34
|
+
const devRow = new Map(d.checks.prepass.rows.map(r => [r.line, r.row]));
|
|
35
|
+
const q = d.qualityFlags ?? [];
|
|
36
|
+
return [
|
|
37
|
+
`> **${AUTO_LABEL}** — written by the content pipeline from this session; review before publishing.`,
|
|
38
|
+
`# ${s.title || '(untitled)'}`,
|
|
39
|
+
s.hook ? `*${s.hook}*` : '',
|
|
40
|
+
q.length ? `**Draft quality notes:** this draft passed the hard checks but is weak on ${q.length} point(s):\n${q.map(f => `- ${f}`).join('\n')}` : '',
|
|
41
|
+
'## Brief',
|
|
42
|
+
`- **Viewer:** ${b.viewer}`,
|
|
43
|
+
`- **Living through:** ${b.situation}`,
|
|
44
|
+
`- **Tier:** ${TIER_TEMPLATES[b.tier].label} · ~${s.estSeconds}s (${s.words} words)`,
|
|
45
|
+
`- **Format:** ${fmt?.name ?? b.format}`,
|
|
46
|
+
b.stake ? `- **Stake:** ${b.stake.quantity}: ${b.stake.before} → ${b.stake.after}` : '',
|
|
47
|
+
`- **Angle:** ${b.angle}`,
|
|
48
|
+
b.problems.length ? `- **Speaks to:** ${b.problems.join('; ')}` : '',
|
|
49
|
+
'## Script',
|
|
50
|
+
...s.lines.map((l, i) => `${i + 1}. **${LABEL[l.speaker]}:** ${l.text}${l.speaker === 'dev' && devRow.has(i + 1) ? ` _(row #${devRow.get(i + 1)})_` : ''}`),
|
|
51
|
+
d.citations.length ? '## Research threads' : '',
|
|
52
|
+
...d.citations.map(c => `- [${c.title}](${c.url}) (${c.source})`),
|
|
53
|
+
'## Checks',
|
|
54
|
+
`- Dev-voice pre-pass: ${d.checks.prepass.ok ? 'pass' : 'FAIL'} · length ~${d.checks.length.seconds}s (${d.checks.length.min}-${d.checks.length.max}s)`,
|
|
55
|
+
d.checks.truth ? `- Truth-check (${d.checks.truth.model} via ${d.checks.truth.provider}): ${d.checks.truth.flags.length} flag(s)` : '',
|
|
56
|
+
d.checks.audience ? `- Audience check: ${d.checks.audience.flags.length} flag(s)` : '',
|
|
57
|
+
]
|
|
58
|
+
.filter(Boolean)
|
|
59
|
+
.join('\n');
|
|
60
|
+
}
|
|
61
|
+
export function buildScriptPayload(d, scrubDeep = v => v) {
|
|
62
|
+
const s = d.script;
|
|
63
|
+
const b = d.brief;
|
|
64
|
+
const payload = {
|
|
65
|
+
type: 'text_post',
|
|
66
|
+
title: `${AUTO_LABEL}: ${s.title || b.subtopic}`.slice(0, 200),
|
|
67
|
+
hook: s.hook || null,
|
|
68
|
+
body: scriptMarkdown(d),
|
|
69
|
+
transcript: scriptTranscript(s),
|
|
70
|
+
transcript_segments: s.lines.map((l, i) => ({ i: i + 1, speaker: l.speaker, text: l.text, ...(l.row ? { row: l.row } : {}) })),
|
|
71
|
+
source_session_id: SESSION_ID_RE.test(d.sessionId) ? d.sessionId : null,
|
|
72
|
+
source_kind: b.tier === 'highlight' ? 'script_highlight' : 'script_howto',
|
|
73
|
+
source_anchors: {
|
|
74
|
+
auto_generated: true,
|
|
75
|
+
label: AUTO_LABEL,
|
|
76
|
+
project_slug: projectSlugFor(d.projectDir),
|
|
77
|
+
piece_id: b.id,
|
|
78
|
+
period: { index: d.period.index, start_row_id: d.period.startRowId, end_row_id: d.period.endRowId, page: d.period.page },
|
|
79
|
+
brief: {
|
|
80
|
+
viewer: b.viewer, situation: b.situation, tier: b.tier, format: b.format, angle: b.angle, problems: b.problems,
|
|
81
|
+
stake: b.stake, subtopic: b.subtopic, story: b.story, from_row: b.fromRow, to_row: b.toRow, owner: b.owner,
|
|
82
|
+
},
|
|
83
|
+
research: { key: b.researchKey, citations: d.citations },
|
|
84
|
+
checks: { ...d.checks, flags: 0 },
|
|
85
|
+
quality_flags: d.qualityFlags ?? [],
|
|
86
|
+
dev_rows: d.checks.prepass.rows,
|
|
87
|
+
est_seconds: s.estSeconds,
|
|
88
|
+
generator: { model: d.model, check_model: CHECK_MODEL, check_provider: CHECK_PROVIDER },
|
|
89
|
+
},
|
|
90
|
+
content_hash: pieceContentHash(d.sessionId, d.period.startRowId, b.id),
|
|
91
|
+
};
|
|
92
|
+
// Redact every string, then restore the hash (hex, untouched by design but never trust a scrubber with it).
|
|
93
|
+
return { ...scrubDeep(payload), content_hash: payload.content_hash };
|
|
94
|
+
}
|
|
95
|
+
export const isIngestConfigured = () => resolveIngestConfig().ok;
|
|
96
|
+
/** Ingest one checked script draft. Never throws. */
|
|
97
|
+
export async function ingestScript(d, scrubDeep, o = {}) {
|
|
98
|
+
const hash = pieceContentHash(d.sessionId, d.period.startRowId, d.brief.id);
|
|
99
|
+
let payload = null;
|
|
100
|
+
try {
|
|
101
|
+
payload = buildScriptPayload(d, scrubDeep);
|
|
102
|
+
}
|
|
103
|
+
catch {
|
|
104
|
+
return { status: 'error', reason: 'payload', contentHash: hash, at: new Date().toISOString() };
|
|
105
|
+
}
|
|
106
|
+
return postContentIngest(payload, hash, `content: ingest ${d.brief.id} (period ${d.period.index})`, o);
|
|
107
|
+
}
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* content-llm.ts — Stage A's flags, spend caps and its ONE model-call path.
|
|
3
|
+
* OpenRouter ONLY (OPENROUTER_API_KEY) — never the Claude subscription / Anthropic SDK.
|
|
4
|
+
*
|
|
5
|
+
* Flags:
|
|
6
|
+
* OSBORN_CONTENT_PIPELINE default ON; 0/off/false/no = off. OSBORN_CONTENT_LENS=0 also turns it off.
|
|
7
|
+
* Caps (a hit = status "capped", never an error):
|
|
8
|
+
* OSBORN_CONTENT_PERIOD_USD default 0.75 (cumulative per period, kept in the content manifest)
|
|
9
|
+
* OSBORN_CONTENT_DAILY_USD default 2.00 (UTC day, per MACHINE: $OSBORN_HOME or ~/.osborn/content-spend-ledger.json,
|
|
10
|
+
* locked read-modify-write so concurrent workers across projects share one cap)
|
|
11
|
+
* OSBORN_CONTENT_MAX_PIECES default 4 (1 highlight + up to 3 how-tos)
|
|
12
|
+
* Every call reserves its WORST case (prompt + max output) against both caps up
|
|
13
|
+
* front and writes it to the ledger before the fetch; it is reconciled to the
|
|
14
|
+
* actual cost after. A failed call is charged the worst case, and a hard exit
|
|
15
|
+
* mid-call leaves the worst case in the ledger, so the caps stay hard.
|
|
16
|
+
* All outbound prompt text goes through the caller's scrub() first.
|
|
17
|
+
*/
|
|
18
|
+
export declare const LEDGER_FILE = "content-spend-ledger.json";
|
|
19
|
+
export declare const DEFAULT_GEN_MODEL = "minimax/minimax-m3";
|
|
20
|
+
/** Pinned judge (proven in videos-test-3/vt3-render.ts): model + provider endpoint, no fallbacks. */
|
|
21
|
+
export declare const CHECK_MODEL = "deepseek/deepseek-chat";
|
|
22
|
+
export declare const CHECK_PROVIDER = "deepinfra/fp4";
|
|
23
|
+
export declare const CHECK_PROVIDER_NAME = "DeepInfra";
|
|
24
|
+
export declare const priceFor: (model: string) => {
|
|
25
|
+
p: number;
|
|
26
|
+
c: number;
|
|
27
|
+
};
|
|
28
|
+
/** OFF when OSBORN_CONTENT_PIPELINE, OSBORN_CONTENT_LENS or OSBORN_CONTENT_INGEST is 0/off/false/no: ingest off ⇒ no LLM spend at all. */
|
|
29
|
+
export declare function isContentPipelineEnabled(): boolean;
|
|
30
|
+
/**
|
|
31
|
+
* OSBORN_CONTENT_STRICT=1 restores hard blocks for the quality gates (structure,
|
|
32
|
+
* over-length tolerance, missing stake, audience). Default: those are advisory —
|
|
33
|
+
* the piece is still scripted and ingested as a draft with quality_flags attached.
|
|
34
|
+
* Dev-voice, truth-check, redaction, the outbound-query gate and the caps are hard either way.
|
|
35
|
+
*/
|
|
36
|
+
export declare function isContentStrict(): boolean;
|
|
37
|
+
export interface ContentCaps {
|
|
38
|
+
periodUsd: number;
|
|
39
|
+
dailyUsd: number;
|
|
40
|
+
maxPieces: number;
|
|
41
|
+
}
|
|
42
|
+
export declare function contentCaps(): ContentCaps;
|
|
43
|
+
export declare const genModel: () => string;
|
|
44
|
+
/** Machine-scoped ledger dir: $OSBORN_HOME, else ~/.osborn. The DAY cap is per machine, not per project. */
|
|
45
|
+
export declare function ledgerDir(): string;
|
|
46
|
+
export declare const ledgerPath: () => string;
|
|
47
|
+
interface Ledger {
|
|
48
|
+
days: Record<string, number>;
|
|
49
|
+
}
|
|
50
|
+
/** The machine ledger (UTC day → USD). */
|
|
51
|
+
export declare function readLedger(): Ledger;
|
|
52
|
+
/** Spend recorded (incl. open reservations) on this machine today, across every project. */
|
|
53
|
+
export declare function daySpent(now?: number): number;
|
|
54
|
+
export type CapKind = 'period' | 'day' | 'time';
|
|
55
|
+
export declare class CapError extends Error {
|
|
56
|
+
readonly cap: CapKind;
|
|
57
|
+
readonly needUsd: number;
|
|
58
|
+
constructor(cap: CapKind, needUsd: number);
|
|
59
|
+
}
|
|
60
|
+
/** A worst-case amount already written to the machine ledger; settle() reconciles it to the actual cost. */
|
|
61
|
+
export interface Reservation {
|
|
62
|
+
day: string;
|
|
63
|
+
usd: number;
|
|
64
|
+
inLedger: boolean;
|
|
65
|
+
settled: boolean;
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Per-run spend guard: the per-period cap (cumulative, per project, in memory +
|
|
69
|
+
* content manifest), the per-day cap (per MACHINE, in the locked ledger) and an
|
|
70
|
+
* optional wall-clock deadline after which no new call starts.
|
|
71
|
+
*/
|
|
72
|
+
export declare class SpendGuard {
|
|
73
|
+
/** Kept for callers/logs; the day ledger is machine-scoped, not under this dir. */
|
|
74
|
+
readonly projectDir: string | null;
|
|
75
|
+
readonly caps: ContentCaps;
|
|
76
|
+
/** Spend already recorded for this period by earlier runs. */
|
|
77
|
+
periodUsd: number;
|
|
78
|
+
/** Epoch ms after which reserve() refuses new calls (CapError 'time'). */
|
|
79
|
+
readonly deadline: number | null;
|
|
80
|
+
runUsd: number;
|
|
81
|
+
calls: number;
|
|
82
|
+
capHit: CapKind | null;
|
|
83
|
+
constructor(
|
|
84
|
+
/** Kept for callers/logs; the day ledger is machine-scoped, not under this dir. */
|
|
85
|
+
projectDir: string | null, caps: ContentCaps,
|
|
86
|
+
/** Spend already recorded for this period by earlier runs. */
|
|
87
|
+
periodUsd?: number,
|
|
88
|
+
/** Epoch ms after which reserve() refuses new calls (CapError 'time'). */
|
|
89
|
+
deadline?: number | null);
|
|
90
|
+
/**
|
|
91
|
+
* Reserve a worst-case spend of `usd`: throws CapError if it would cross a cap
|
|
92
|
+
* (or the deadline passed); otherwise writes it to the machine ledger BEFORE the
|
|
93
|
+
* call, so a hard exit mid-call still counts it. Fails closed if the ledger lock
|
|
94
|
+
* cannot be taken.
|
|
95
|
+
*/
|
|
96
|
+
reserve(usd: number): Reservation;
|
|
97
|
+
private hit;
|
|
98
|
+
/** Reconcile a reservation to the actual cost (once). The in-memory totals always move; the ledger is best-effort. */
|
|
99
|
+
settle(r: Reservation, actualUsd: number): void;
|
|
100
|
+
}
|
|
101
|
+
export interface LlmCall {
|
|
102
|
+
/** Short tag for logs ("research", "script", "truth-check"...). Never the prompt. */
|
|
103
|
+
what: string;
|
|
104
|
+
system: string;
|
|
105
|
+
user: string;
|
|
106
|
+
maxOut: number;
|
|
107
|
+
model?: string;
|
|
108
|
+
temperature?: number;
|
|
109
|
+
/** Pin provider: order=[endpoint], allow_fallbacks:false; the served provider must equal `name`. */
|
|
110
|
+
pin?: {
|
|
111
|
+
endpoint: string;
|
|
112
|
+
name: string;
|
|
113
|
+
};
|
|
114
|
+
}
|
|
115
|
+
export interface LlmResult {
|
|
116
|
+
content: string;
|
|
117
|
+
json: any;
|
|
118
|
+
costUsd: number;
|
|
119
|
+
provider: string;
|
|
120
|
+
model: string;
|
|
121
|
+
finish: string;
|
|
122
|
+
}
|
|
123
|
+
/** Lenient JSON extraction: strips fences, takes the outermost {...} or [...]. Null when unparseable. */
|
|
124
|
+
export declare function parseJsonLoose(s: string): any;
|
|
125
|
+
export interface ContentLlmOptions {
|
|
126
|
+
apiKey: string;
|
|
127
|
+
guard: SpendGuard;
|
|
128
|
+
scrub: (s: string) => string;
|
|
129
|
+
fetchImpl?: typeof fetch;
|
|
130
|
+
log?: (m: string) => void;
|
|
131
|
+
}
|
|
132
|
+
/** One chat call → JSON. Throws CapError on a cap, Error on HTTP/timeout/provider mismatch. */
|
|
133
|
+
export declare function llmJson(o: ContentLlmOptions, call: LlmCall): Promise<LlmResult>;
|
|
134
|
+
export {};
|