@growth-labs/cms 0.8.26 → 0.8.30

package/src/schema/types.ts

@@ -611,6 +611,16 @@ export interface InsightBrief {
 	cluster: {
 		canonical: string
 		spokes: string[]
+		selected?: {
+			site_id: string
+			run_id: string
+			intent_key: string
+			desk: string
+			decision_ref: string
+			taxonomy_ref: string
+			hub: { content_id: string; revision_id: string; url: string; title: string }
+			children: Array<{ content_id: string; revision_id: string; url: string; title: string }>
+		}
 	}
 	sources: Array<{ title: string; url: string }>
 	format?: 'cornerstone' | 'pillar' | 'standalone'

package/src/surveys/auto-label.ts

@@ -11,9 +11,15 @@
 // pre-filter plus <answer> delimiting so personal details never reach the
 // model and reader text is framed as data, not instructions (B2).
 
+import { z } from 'zod'
 import type { D1Database } from '../engine/d1.js'
-import type { LlmDispatch, LlmDispatchResult } from '../providers/types.js'
-import type { SurveyAnswerLabel } from './schema.js'
+import type { LlmDispatch, LlmDispatchResult, LlmDispatchSubmission } from '../providers/types.js'
+import {
+	BoundedRecordIdSchema,
+	type SurveyAnswerLabel,
+	SurveyIdSchema,
+	SurveyQuestionIdSchema,
+} from './schema.js'
 
 /** The masthead-ai-assist task for survey classification. Foundry does not
  *  validate `task` (articlechain/mastheadaiassist.go: "a sixth Masthead action
@@ -146,6 +152,9 @@ export interface SurveyAutoLabelAttemptRecord {
 	nextAt: number
 	/** Epoch ms until another request's dispatch claims this answer. */
 	inflightUntil: number
+	/** Opaque Go correlation, never reader text or identity. */
+	taskId?: string
+	reason?: 'callback_expired' | 'dispatch_failed' | 'llm_failed' | 'unparseable'
 }
 
 /** Narrow persistence the batch runner needs; the KV adapter below satisfies
@@ -154,6 +163,8 @@ export interface SurveyAutoLabelAttemptStore {
 	get(key: string): Promise<SurveyAutoLabelAttemptRecord | null>
 	put(key: string, record: SurveyAutoLabelAttemptRecord, ttlSeconds: number): Promise<void>
 	delete(key: string): Promise<void>
+	putTask?(taskId: string, record: SurveyAutoLabelTaskRecord): Promise<void>
+	getTask?(taskId: string): Promise<SurveyAutoLabelTaskRecord | null>
 }
 
 export function surveyAutoLabelAttemptKey(
@@ -175,31 +186,111 @@ interface KvLike {
 /** Attempt records in a Workers KV namespace (`survey-autolabel:` prefix).
  *  The same namespace the host's LlmDispatch result slots live in is fine:
  *  the key prefixes never collide (`masthead:llm:result:` vs ours). */
+const AttemptRecordSchema = z
+	.object({
+		attempts: z.number().int().nonnegative(),
+		nextAt: z.number().nonnegative(),
+		inflightUntil: z.number().nonnegative(),
+		taskId: z.string().min(1).max(256).optional(),
+		reason: z.enum(['callback_expired', 'dispatch_failed', 'llm_failed', 'unparseable']).optional(),
+	})
+	.strict()
+const TaskRecordSchema = z
+	.object({
+		surveyId: SurveyIdSchema,
+		responseId: BoundedRecordIdSchema,
+		questionId: SurveyQuestionIdSchema,
+		attempts: z.number().int().nonnegative(),
+	})
+	.strict()
+export type SurveyAutoLabelTaskRecord = z.infer<typeof TaskRecordSchema>
+export const SURVEY_AUTO_LABEL_TASK_PREFIX = 'survey-autolabel-task:'
+
 export function kvSurveyAutoLabelAttemptStore(kv: KvLike): SurveyAutoLabelAttemptStore {
 	return {
 		async get(key) {
 			const raw = await kv.get(key)
-			if (raw === null) return null
-			try {
-				const value = JSON.parse(raw) as Partial<SurveyAutoLabelAttemptRecord>
-				return {
-					attempts: typeof value.attempts === 'number' ? value.attempts : 0,
-					nextAt: typeof value.nextAt === 'number' ? value.nextAt : 0,
-					inflightUntil: typeof value.inflightUntil === 'number' ? value.inflightUntil : 0,
-				}
-			} catch {
-				return null
-			}
+			return raw === null ? null : AttemptRecordSchema.parse(JSON.parse(raw))
 		},
 		async put(key, record, ttlSeconds) {
-			await kv.put(key, JSON.stringify(record), { expirationTtl: ttlSeconds })
+			await kv.put(key, JSON.stringify(AttemptRecordSchema.parse(record)), {
+				expirationTtl: ttlSeconds,
+			})
 		},
 		async delete(key) {
 			await kv.delete(key)
 		},
+		async putTask(taskId, record) {
+			await kv.put(
+				`${SURVEY_AUTO_LABEL_TASK_PREFIX}${taskId}`,
+				JSON.stringify(TaskRecordSchema.parse(record)),
+				{ expirationTtl: SURVEY_AUTO_LABEL_ATTEMPT_TTL_SECONDS },
+			)
+		},
+		async getTask(taskId) {
+			const raw = await kv.get(`${SURVEY_AUTO_LABEL_TASK_PREFIX}${taskId}`)
+			return raw === null ? null : TaskRecordSchema.parse(JSON.parse(raw))
+		},
 	}
 }
 
+/** Finalize through the existing signed callback, independently of waitUntil.
+ * Missing correlation/persistence throws so Go retries rather than acknowledging lost work. */
+export async function finalizeSurveyAutoLabel(
+	db: D1Database,
+	store: SurveyAutoLabelAttemptStore,
+	taskId: string,
+	result: { terminalState: 'completed' | 'failed'; result?: string },
+	now = Date.now(),
+): Promise<void> {
+	const target = await store.getTask?.(taskId)
+	if (!target) throw new Error('survey correlation unavailable')
+	const key = surveyAutoLabelAttemptKey(target.surveyId, target.responseId, target.questionId)
+	const label =
+		result.terminalState === 'completed' ? parseSurveyLabelReply(result.result ?? '') : null
+	if (label) {
+		await db
+			.prepare(`INSERT INTO survey_answer_labels
+		 (survey_id, response_id, question_id, label, source, labeled_by, labeled_at)
+		 VALUES (?, ?, ?, ?, 'auto', NULL, ?)
+		 ON CONFLICT(survey_id, response_id, question_id) DO NOTHING`)
+			.bind(target.surveyId, target.responseId, target.questionId, label, now)
+			.run()
+		const current = await store.get(key)
+		if (current?.taskId === taskId) await store.delete(key)
+		return
+	}
+	const existing = await db
+		.prepare(
+			'SELECT label FROM survey_answer_labels WHERE survey_id = ? AND response_id = ? AND question_id = ? LIMIT 1',
+		)
+		.bind(target.surveyId, target.responseId, target.questionId)
+		.first()
+	if (existing) return
+	const current = await store.get(key)
+	if (!current) throw new Error('survey attempt unavailable')
+	if (
+		current.taskId === taskId &&
+		current.inflightUntil === 0 &&
+		current.attempts > target.attempts
+	)
+		return
+	// A delayed failed callback must not reset a newer retry or a completed label.
+	if (current?.taskId !== taskId) return
+	const attempts = target.attempts + 1
+	await store.put(
+		key,
+		{
+			attempts,
+			nextAt: now + SURVEY_AUTO_LABEL_BACKOFF_MS[Math.min(attempts, 3) - 1],
+			inflightUntil: 0,
+			taskId,
+			reason: result.terminalState === 'failed' ? 'llm_failed' : 'unparseable',
+		},
+		SURVEY_AUTO_LABEL_ATTEMPT_TTL_SECONDS,
+	)
+}
+
 /**
  * In-isolate in-flight gate. KV has no compare-and-swap, so a dedupe window
  * this small is enforced synchronously in module scope (all concurrent
@@ -256,18 +347,34 @@ async function runSurveyAutoLabelBatch(
 	batch: SurveyAutoLabelCandidate[],
 ): Promise<void> {
 	let cursor = 0
+	let admitted = 0
+	const reserveAdmission = () => {
+		if (admitted >= SURVEY_AUTO_LABEL_BATCH_LIMIT) return false
+		admitted += 1
+		return true
+	}
 	let stored = 0
+	let processed = 0
 	let deferred = 0
 	let halted: string | null = null
+	const admissionDeadline = (options.now ?? Date.now)() + 20_000
 	const skipReasons = new Set<string>()
 
 	const worker = async (): Promise<void> => {
 		for (;;) {
-			if (halted !== null) return
+			if (halted !== null || admitted >= SURVEY_AUTO_LABEL_BATCH_LIMIT) return
+			if ((options.now ?? Date.now)() >= admissionDeadline) {
+				halted = 'admission_window_ended'
+				return
+			}
 			const next = cursor
 			cursor += 1
 			if (next >= batch.length) return
-			const outcome = await classifyAndStore(options, batch[next])
+			const outcome = await classifyAndStore(options, batch[next], reserveAdmission).catch(() => ({
+				kind: 'unavailable' as const,
+				reason: 'attempt_write_failed',
+			}))
+			processed += 1
 			if (outcome.kind === 'stored') {
 				stored += 1
 				continue
@@ -292,7 +399,7 @@ async function runSurveyAutoLabelBatch(
 
 	// Failure-class skips only: a wholly deferred batch (all in flight or
 	// backing off) is the steady state of the dedupe, not a failure to report.
-	const skipped = batch.length - stored - deferred
+	const skipped = (halted === null ? processed : batch.length) - stored - deferred
 	if (skipped === 0) return
 	const reasons = new Set(skipReasons)
 	if (halted !== null) reasons.add(halted)
@@ -305,14 +412,19 @@ async function runSurveyAutoLabelBatch(
 
 type ClassifyOutcome =
 	| { kind: 'stored' }
-	| { kind: 'deferred'; reason: 'in_flight' | 'backoff' | 'gave_up' | 'pii' }
+	| {
+			kind: 'deferred'
+			reason: 'in_flight' | 'backoff' | 'gave_up' | 'pii' | 'empty' | 'batch_limit'
+	  }
 	| { kind: 'skipped'; reason: string }
 	| { kind: 'unavailable'; reason: string }
 
 async function classifyAndStore(
 	options: SurveyAutoLabelOptions,
 	candidate: SurveyAutoLabelCandidate,
+	reserveAdmission: () => boolean,
 ): Promise<ClassifyOutcome> {
+	if (candidate.answerText.trim().length === 0) return { kind: 'deferred', reason: 'empty' }
 	// B2 first, before any store or dispatch work: personal details never
 	// leave the Worker, and the skip consumes no attempt.
 	if (containsPersonalDetails(candidate.answerText)) return { kind: 'deferred', reason: 'pii' }
@@ -327,12 +439,25 @@ async function classifyAndStore(
 	try {
 		record = await options.attempts.get(key)
 	} catch {
-		// The attempt store is dedupe hygiene, not label correctness: an
-		// unreadable record proceeds as if absent (worst case: one duplicate
-		// dispatch, still bounded by the in-isolate gate).
+		return { kind: 'unavailable', reason: 'attempt_read_failed' }
 	}
 	if (record) {
 		if (record.inflightUntil > now) return { kind: 'deferred', reason: 'in_flight' }
+		if (record.inflightUntil > 0) {
+			const attempts = record.attempts + 1
+			await options.attempts.put(
+				key,
+				{
+					attempts,
+					nextAt: now + SURVEY_AUTO_LABEL_BACKOFF_MS[Math.min(attempts, 3) - 1],
+					inflightUntil: 0,
+					...(record.taskId ? { taskId: record.taskId } : {}),
+					reason: 'callback_expired',
+				},
+				SURVEY_AUTO_LABEL_ATTEMPT_TTL_SECONDS,
+			)
+			return { kind: 'deferred', reason: 'backoff' }
+		}
 		if (record.attempts >= SURVEY_AUTO_LABEL_MAX_ATTEMPTS) {
 			return { kind: 'deferred', reason: 'gave_up' }
 		}
@@ -340,6 +465,7 @@ async function classifyAndStore(
 	}
 	if (!reserveInFlight(key, now)) return { kind: 'deferred', reason: 'in_flight' }
 	try {
+		if (!reserveAdmission()) return { kind: 'deferred', reason: 'batch_limit' }
 		// Claimed. Record the failure (or the backoff it earns) at each exit;
 		// the KV marker goes out BEFORE the dispatch so a concurrent request
 		// in another isolate defers instead of duplicating the work.
@@ -352,23 +478,68 @@ async function classifyAndStore(
 			await options.attempts
 				.put(
 					key,
-					{ attempts, nextAt: now + backoff, inflightUntil: 0 },
+					{
+						attempts,
+						nextAt: now + backoff,
+						inflightUntil: 0,
+						reason: outcome.reason === 'unparseable' ? 'unparseable' : 'dispatch_failed',
+					},
 					SURVEY_AUTO_LABEL_ATTEMPT_TTL_SECONDS,
 				)
 				.catch(() => {})
 			return outcome
 		}
-		await options.attempts
-			.put(
-				key,
-				{
-					attempts: record?.attempts ?? 0,
-					nextAt: record?.nextAt ?? 0,
-					inflightUntil: now + SURVEY_AUTO_LABEL_INFLIGHT_MS,
-				},
-				SURVEY_AUTO_LABEL_ATTEMPT_TTL_SECONDS,
-			)
-			.catch(() => {})
+		if (!options.llm.submit)
+			await options.attempts
+				.put(
+					key,
+					{
+						attempts: record?.attempts ?? 0,
+						nextAt: record?.nextAt ?? 0,
+						inflightUntil: now + SURVEY_AUTO_LABEL_INFLIGHT_MS,
+					},
+					SURVEY_AUTO_LABEL_ATTEMPT_TTL_SECONDS,
+				)
+				.catch(() => {
+					throw new Error('attempt_write_failed')
+				})
+
+		if (options.llm.submit) {
+			if (!options.attempts.putTask)
+				return await failure({ kind: 'unavailable', reason: 'no_correlation_store' })
+			let admission: LlmDispatchSubmission
+			try {
+				admission = await options.llm.submit(
+					{
+						task: SURVEY_AUTO_LABEL_TASK,
+						prompt: buildSurveyLabelPrompt(candidate.questionPrompt, candidate.answerText),
+					},
+					async (taskId) => {
+						await options.attempts.putTask!(taskId, {
+							surveyId: options.surveyId,
+							responseId: candidate.responseId,
+							questionId: candidate.questionId,
+							attempts: record?.attempts ?? 0,
+						})
+						await options.attempts.put(
+							key,
+							{
+								attempts: record?.attempts ?? 0,
+								nextAt: 0,
+								inflightUntil: now + SURVEY_AUTO_LABEL_INFLIGHT_MS,
+								taskId,
+							},
+							SURVEY_AUTO_LABEL_ATTEMPT_TTL_SECONDS,
+						)
+					},
+				)
+			} catch {
+				return await failure({ kind: 'unavailable', reason: 'dispatch_threw' })
+			}
+			if (admission.status === 'unavailable')
+				return await failure({ kind: 'unavailable', reason: admission.reason })
+			return { kind: 'deferred', reason: 'in_flight' }
+		}
 
 		let result: LlmDispatchResult
 		try {

package/src/surveys/schema.ts

@@ -196,6 +196,38 @@ export const SurveyQuestionSchema = z
 		}
 	})
 
+/** Reviewed delivery notes supplied by a site, never generated from private answers. */
+export const SurveyFeedbackThemeSchema = z
+	.object({
+		id: SurveyIdSchema,
+		title: z.string().trim().min(1).max(160),
+		state: z.enum(['open', 'partly_shipped', 'shipped', 'unknown']),
+		summary: z.string().trim().min(1).max(1_000),
+		reviewedAt: IsoTimestampSchema,
+		evidence: z
+			.array(
+				z
+					.object({
+						label: z.string().trim().min(1).max(160),
+						href: z
+							.string()
+							.url()
+							.max(2_000)
+							.refine((value) => value.startsWith('https://')),
+					})
+					.strict(),
+			)
+			.max(6),
+	})
+	.strict()
+export const SurveyFeedbackThemesSchema = z
+	.array(SurveyFeedbackThemeSchema)
+	.max(25)
+	.refine((themes) => new Set(themes.map(({ id }) => id)).size === themes.length, {
+		message: 'Feedback theme IDs must be unique',
+	})
+export type SurveyFeedbackTheme = z.infer<typeof SurveyFeedbackThemeSchema>
+
 export const SurveyResultsSchema = z
 	.object({
 		survey: z
@@ -244,6 +276,28 @@ export const SurveyResultsSchema = z
 									.object({
 										questionId: SurveyQuestionIdSchema,
 										value: AnswerValueSchema,
+										writtenValue: z
+											.string()
+											.max(
+												Math.max(
+													MAX_FREE_TEXT_LENGTH,
+													MAX_CHOICE_OPTIONS * (MAX_OPTION_LENGTH + 1),
+												),
+											)
+											.optional(),
+										label: z.enum(['technical', 'general']).optional(),
+										labelSource: z.enum(['owner', 'auto', 'import']).optional(),
+										classificationStatus: z
+											.enum([
+												'pending',
+												'retry',
+												'expired',
+												'manual_review',
+												'unavailable',
+												'queued',
+											])
+											.optional(),
+										nextAttemptAt: z.number().nonnegative().optional(),
 									})
 									.strict(),
 							)
@@ -269,6 +323,10 @@ export const SurveyResultsSchema = z
 						// Owner payloads carry it; non-owner entries pass through
 						// undecorated, exactly as they do for `label`.
 						labelSource: z.enum(['owner', 'auto', 'import']).optional(),
+						classificationStatus: z
+							.enum(['pending', 'retry', 'expired', 'manual_review', 'unavailable', 'queued'])
+							.optional(),
+						nextAttemptAt: z.number().nonnegative().optional(),
 					})
 					.strict(),
 			)

package/src/ui/api/content/insights-inputs.ts

@@ -4,10 +4,13 @@
 // X-Foundry-Timestamp + X-Foundry-Signature family as fronts/schedule) for
 // foundryd's sitebrain.insights.run schedule (W1-07 PR 6). NOT behind
 // requirePublisher. Sibling of insights-ingest under /content/.
+
+import { config } from 'virtual:growth-labs/cms/config'
 import type { APIContext } from 'astro'
 import { createContentInsightsRoutes } from '../../../routes/content-insights.js'
 import type { RouteContext } from '../../../routes/context.js'
 import { buildSiteAuthz } from '../_authz.js'
+import { toContentConfig } from '../_content-config.js'
 
 function toCtx(c: APIContext): RouteContext {
 	const env = (c.locals as { cmsEnv?: Record<string, unknown> }).cmsEnv ?? {}
@@ -16,7 +19,11 @@ function toCtx(c: APIContext): RouteContext {
 
 function siteId(c: APIContext): string {
 	const env = (c.locals as { cmsEnv?: Record<string, unknown> }).cmsEnv ?? {}
-	return (env.ACTIVE_WORKSPACE_ID as string | undefined) ?? 'default'
+	return (
+		toContentConfig(c).activeWorkspaceId ??
+		(env.ACTIVE_WORKSPACE_ID as string | undefined) ??
+		(config.workspaces?.length === 1 ? config.workspaces[0]!.id : 'default')
+	)
 }
 
 export const GET = (c: APIContext) =>

package/src/ui/client/mount.tsx

@@ -42,6 +42,7 @@ export async function mount(elementId = 'masthead-root') {
 		primaryTopic: config.primaryTopic,
 		contentMetadataSelects: config.contentMetadataSelects,
 		surveyId: config.surveyResults?.surveyId,
+		surveys: config.surveyResults?.surveys,
 	})
 	createRoot(el).render(<CmsApp initial={initial} screens={screens} boot={boot} />)
 }