@growth-labs/cms 0.8.20 → 0.8.24

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (182) hide show
  1. package/dist/cli/foundry-emit.d.ts +14 -0
  2. package/dist/cli/foundry-emit.d.ts.map +1 -0
  3. package/dist/cli/foundry-emit.js +69 -0
  4. package/dist/cli/foundry-emit.js.map +1 -0
  5. package/dist/engine/foundry-dispatch.d.ts +1 -0
  6. package/dist/engine/foundry-dispatch.d.ts.map +1 -1
  7. package/dist/engine/foundry-dispatch.js +1 -1
  8. package/dist/engine/foundry-dispatch.js.map +1 -1
  9. package/dist/engine/fronts-publish-intent.d.ts +1 -1
  10. package/dist/engine/fronts-publish-intent.d.ts.map +1 -1
  11. package/dist/engine/fronts-publish-intent.js +12 -1
  12. package/dist/engine/fronts-publish-intent.js.map +1 -1
  13. package/dist/engine/index.d.ts +1 -0
  14. package/dist/engine/index.d.ts.map +1 -1
  15. package/dist/engine/index.js +2 -0
  16. package/dist/engine/index.js.map +1 -1
  17. package/dist/engine/publication-receipt/receipt.d.ts +20 -20
  18. package/dist/engine/publisher.d.ts +16 -1
  19. package/dist/engine/publisher.d.ts.map +1 -1
  20. package/dist/engine/publisher.js +55 -4
  21. package/dist/engine/publisher.js.map +1 -1
  22. package/dist/engine/tags.d.ts.map +1 -1
  23. package/dist/engine/tags.js +18 -1
  24. package/dist/engine/tags.js.map +1 -1
  25. package/dist/engine/topics.d.ts.map +1 -1
  26. package/dist/engine/topics.js +4 -1
  27. package/dist/engine/topics.js.map +1 -1
  28. package/dist/engine/video-source.d.ts +21 -0
  29. package/dist/engine/video-source.d.ts.map +1 -1
  30. package/dist/engine/video-source.js +40 -0
  31. package/dist/engine/video-source.js.map +1 -1
  32. package/dist/foundry-emit.d.ts +146 -0
  33. package/dist/foundry-emit.d.ts.map +1 -0
  34. package/dist/foundry-emit.js +189 -0
  35. package/dist/foundry-emit.js.map +1 -0
  36. package/dist/integration/index.d.ts.map +1 -1
  37. package/dist/integration/index.js +9 -0
  38. package/dist/integration/index.js.map +1 -1
  39. package/dist/integration/options.d.ts +24 -0
  40. package/dist/integration/options.d.ts.map +1 -1
  41. package/dist/integration/options.js +22 -0
  42. package/dist/integration/options.js.map +1 -1
  43. package/dist/providers/null.d.ts.map +1 -1
  44. package/dist/providers/null.js +6 -0
  45. package/dist/providers/null.js.map +1 -1
  46. package/dist/providers/types.d.ts +56 -0
  47. package/dist/providers/types.d.ts.map +1 -1
  48. package/dist/providers/types.js.map +1 -1
  49. package/dist/routes/ai.d.ts.map +1 -1
  50. package/dist/routes/ai.js +21 -13
  51. package/dist/routes/ai.js.map +1 -1
  52. package/dist/routes/analytics.d.ts +1 -0
  53. package/dist/routes/analytics.d.ts.map +1 -1
  54. package/dist/routes/analytics.js +20 -0
  55. package/dist/routes/analytics.js.map +1 -1
  56. package/dist/routes/content.d.ts +27 -0
  57. package/dist/routes/content.d.ts.map +1 -1
  58. package/dist/routes/content.js +58 -12
  59. package/dist/routes/content.js.map +1 -1
  60. package/dist/routes/foundry-publish.d.ts +50 -0
  61. package/dist/routes/foundry-publish.d.ts.map +1 -1
  62. package/dist/routes/foundry-publish.js +540 -27
  63. package/dist/routes/foundry-publish.js.map +1 -1
  64. package/dist/routes/machine-context.d.ts +6 -0
  65. package/dist/routes/machine-context.d.ts.map +1 -0
  66. package/dist/routes/machine-context.js +20 -0
  67. package/dist/routes/machine-context.js.map +1 -0
  68. package/dist/routes/surveys.d.ts +14 -1
  69. package/dist/routes/surveys.d.ts.map +1 -1
  70. package/dist/routes/surveys.js +515 -10
  71. package/dist/routes/surveys.js.map +1 -1
  72. package/dist/schema/insights-ingest.d.ts +8 -8
  73. package/dist/schema/migrations.d.ts.map +1 -1
  74. package/dist/schema/migrations.js +15 -0
  75. package/dist/schema/migrations.js.map +1 -1
  76. package/dist/schema/tables.d.ts +1 -1
  77. package/dist/schema/tables.d.ts.map +1 -1
  78. package/dist/schema/tables.js +2 -0
  79. package/dist/schema/tables.js.map +1 -1
  80. package/dist/surveys/access.d.ts +1 -0
  81. package/dist/surveys/access.d.ts.map +1 -1
  82. package/dist/surveys/access.js +13 -4
  83. package/dist/surveys/access.js.map +1 -1
  84. package/dist/surveys/auto-label.d.ts +100 -0
  85. package/dist/surveys/auto-label.d.ts.map +1 -0
  86. package/dist/surveys/auto-label.js +325 -0
  87. package/dist/surveys/auto-label.js.map +1 -0
  88. package/dist/surveys/schema.d.ts +38 -12
  89. package/dist/surveys/schema.d.ts.map +1 -1
  90. package/dist/surveys/schema.js +18 -1
  91. package/dist/surveys/schema.js.map +1 -1
  92. package/dist/ui/api/_content-config.d.ts.map +1 -1
  93. package/dist/ui/api/_content-config.js +4 -0
  94. package/dist/ui/api/_content-config.js.map +1 -1
  95. package/dist/ui/api/analytics.d.ts.map +1 -1
  96. package/dist/ui/api/analytics.js +5 -2
  97. package/dist/ui/api/analytics.js.map +1 -1
  98. package/dist/ui/api/foundry/articles/[external_id]/publish.d.ts +3 -0
  99. package/dist/ui/api/foundry/articles/[external_id]/publish.d.ts.map +1 -0
  100. package/dist/ui/api/foundry/articles/[external_id]/publish.js +14 -0
  101. package/dist/ui/api/foundry/articles/[external_id]/publish.js.map +1 -0
  102. package/dist/ui/api/surveys/[surveyId]/answers/label.d.ts +3 -0
  103. package/dist/ui/api/surveys/[surveyId]/answers/label.d.ts.map +1 -0
  104. package/dist/ui/api/surveys/[surveyId]/answers/label.js +17 -0
  105. package/dist/ui/api/surveys/[surveyId]/answers/label.js.map +1 -0
  106. package/dist/ui/api/surveys/[surveyId].d.ts.map +1 -1
  107. package/dist/ui/api/surveys/[surveyId].js +28 -1
  108. package/dist/ui/api/surveys/[surveyId].js.map +1 -1
  109. package/dist/ui/commands.d.ts.map +1 -1
  110. package/dist/ui/commands.js +2 -0
  111. package/dist/ui/commands.js.map +1 -1
  112. package/dist/ui/hash-router.d.ts +1 -1
  113. package/dist/ui/hash-router.d.ts.map +1 -1
  114. package/dist/ui/hash-router.js +1 -0
  115. package/dist/ui/hash-router.js.map +1 -1
  116. package/dist/ui/nav.d.ts.map +1 -1
  117. package/dist/ui/nav.js +1 -0
  118. package/dist/ui/nav.js.map +1 -1
  119. package/dist/ui/screens/AnalyticsScreen.d.ts +6 -1
  120. package/dist/ui/screens/AnalyticsScreen.d.ts.map +1 -1
  121. package/dist/ui/screens/AnalyticsScreen.js +13 -8
  122. package/dist/ui/screens/AnalyticsScreen.js.map +1 -1
  123. package/dist/ui/screens/PodcastListeningScreen.d.ts +6 -0
  124. package/dist/ui/screens/PodcastListeningScreen.d.ts.map +1 -0
  125. package/dist/ui/screens/PodcastListeningScreen.js +244 -0
  126. package/dist/ui/screens/PodcastListeningScreen.js.map +1 -0
  127. package/dist/ui/screens/SurveyResponsesScreen.d.ts +4 -2
  128. package/dist/ui/screens/SurveyResponsesScreen.d.ts.map +1 -1
  129. package/dist/ui/screens/SurveyResponsesScreen.js +71 -12
  130. package/dist/ui/screens/SurveyResponsesScreen.js.map +1 -1
  131. package/dist/ui/screens/analytics-data.d.ts +14 -0
  132. package/dist/ui/screens/analytics-data.d.ts.map +1 -1
  133. package/dist/ui/screens/analytics-data.js +16 -0
  134. package/dist/ui/screens/analytics-data.js.map +1 -1
  135. package/dist/ui/screens/podcast-listening-data.d.ts +19 -0
  136. package/dist/ui/screens/podcast-listening-data.d.ts.map +1 -0
  137. package/dist/ui/screens/podcast-listening-data.js +98 -0
  138. package/dist/ui/screens/podcast-listening-data.js.map +1 -0
  139. package/dist/ui/screens/registry.d.ts.map +1 -1
  140. package/dist/ui/screens/registry.js +2 -0
  141. package/dist/ui/screens/registry.js.map +1 -1
  142. package/docs/foundry-publish.md +208 -0
  143. package/migrations/0033_survey_answer_labels.sql +15 -0
  144. package/package.json +9 -3
  145. package/src/cli/foundry-emit.ts +85 -0
  146. package/src/engine/foundry-dispatch.ts +1 -1
  147. package/src/engine/fronts-publish-intent.ts +23 -1
  148. package/src/engine/index.ts +5 -0
  149. package/src/engine/publisher.ts +80 -4
  150. package/src/engine/tags.ts +21 -1
  151. package/src/engine/topics.ts +4 -1
  152. package/src/engine/video-source.ts +44 -0
  153. package/src/foundry-emit.ts +255 -0
  154. package/src/integration/index.ts +9 -0
  155. package/src/integration/options.ts +23 -0
  156. package/src/providers/null.ts +6 -0
  157. package/src/providers/types.ts +52 -0
  158. package/src/routes/ai.ts +29 -17
  159. package/src/routes/analytics.ts +21 -0
  160. package/src/routes/content.ts +81 -15
  161. package/src/routes/foundry-publish.ts +746 -29
  162. package/src/routes/machine-context.ts +24 -0
  163. package/src/routes/surveys.ts +587 -12
  164. package/src/schema/migrations.ts +15 -0
  165. package/src/schema/tables.ts +2 -0
  166. package/src/surveys/access.ts +14 -4
  167. package/src/surveys/auto-label.ts +420 -0
  168. package/src/surveys/schema.ts +25 -1
  169. package/src/ui/api/_content-config.ts +6 -0
  170. package/src/ui/api/analytics.ts +5 -2
  171. package/src/ui/api/foundry/articles/[external_id]/publish.ts +18 -0
  172. package/src/ui/api/surveys/[surveyId]/answers/label.ts +23 -0
  173. package/src/ui/api/surveys/[surveyId].ts +37 -3
  174. package/src/ui/commands.ts +2 -0
  175. package/src/ui/hash-router.ts +1 -0
  176. package/src/ui/nav.ts +1 -0
  177. package/src/ui/screens/AnalyticsScreen.tsx +34 -6
  178. package/src/ui/screens/PodcastListeningScreen.tsx +529 -0
  179. package/src/ui/screens/SurveyResponsesScreen.tsx +179 -13
  180. package/src/ui/screens/analytics-data.ts +18 -0
  181. package/src/ui/screens/podcast-listening-data.ts +114 -0
  182. package/src/ui/screens/registry.tsx +2 -0
@@ -0,0 +1,24 @@
1
+ // The Foundry machine marker (M3-18 hardening). Internal: deliberately NOT
2
+ // exported from the package entry or any subpath, so only code inside this
3
+ // package can mark a context.
4
+ //
5
+ // routes/foundry-publish.ts marks the native contexts it builds, and only after
6
+ // the request's Foundry HMAC has verified. A context derived from a session
7
+ // (an editor in the admin UI, an API key) is never marked, whatever subject or
8
+ // userId it carries, so "is this the machine?" cannot be answered by a string
9
+ // a person could hold.
10
+
11
+ import type { RouteContext } from './context.js'
12
+
13
+ const FOUNDRY_MACHINE_CONTEXTS = new WeakSet<RouteContext>()
14
+
15
+ /** Mark a context the HMAC-verified Foundry route built. Returns it, for chaining. */
16
+ export function markFoundryMachineContext(ctx: RouteContext): RouteContext {
17
+ FOUNDRY_MACHINE_CONTEXTS.add(ctx)
18
+ return ctx
19
+ }
20
+
21
+ /** True only for a context the HMAC-verified Foundry route marked. */
22
+ export function isFoundryMachineContext(ctx: RouteContext): boolean {
23
+ return FOUNDRY_MACHINE_CONTEXTS.has(ctx)
24
+ }
@@ -1,24 +1,59 @@
1
+ import { z } from 'zod'
2
+ import type { D1Database } from '../engine/d1.js'
1
3
  import { nullProviders } from '../providers/null.js'
2
- import { type CmsProviders, ProviderHttpError } from '../providers/types.js'
4
+ import { type CmsProviders, type LlmDispatch, ProviderHttpError } from '../providers/types.js'
5
+ import { canLabelSurveyAnswers } from '../surveys/access.js'
6
+ import {
7
+ containsPersonalDetails,
8
+ SURVEY_AUTO_LABEL_BATCH_LIMIT,
9
+ type SurveyAutoLabelAttemptStore,
10
+ type SurveyAutoLabelCandidate,
11
+ scheduleSurveyAutoLabel,
12
+ } from '../surveys/auto-label.js'
3
13
  import {
4
14
  SURVEY_ERROR_CODE_HEADER,
5
15
  SURVEY_ERROR_DEFINITIONS,
6
16
  type SurveyErrorCode,
7
17
  surveyErrorBody,
8
18
  } from '../surveys/errors.js'
9
- import { SurveyCsvResultSchema, SurveyIdSchema, SurveyResultsSchema } from '../surveys/schema.js'
10
- import type { Authz, Capability, RouteContext } from './context.js'
19
+ import {
20
+ BoundedRecordIdSchema,
21
+ type SurveyAnswerLabel,
22
+ SurveyCsvResultSchema,
23
+ SurveyIdSchema,
24
+ SurveyQuestionIdSchema,
25
+ type SurveyResults,
26
+ SurveyResultsSchema,
27
+ } from '../surveys/schema.js'
28
+ import type { Authz, Capability, CmsUser, RouteContext } from './context.js'
11
29
 
12
30
  export interface SurveyRouteConfig {
13
31
  authz: Authz
14
32
  providers?: Partial<CmsProviders>
33
+ /**
34
+ * Opt-in automatic labeling of unlabeled written answers (integration
35
+ * option `surveys.autoLabel`). The host adapter supplies its LlmDispatch
36
+ * and the attempt/backoff store here only when the option is on; absent
37
+ * means no dispatch ever runs.
38
+ */
39
+ autoLabel?: { llm: LlmDispatch; attempts: SurveyAutoLabelAttemptStore }
15
40
  }
16
41
 
17
42
  export interface SurveyRouteHandlers {
18
43
  results(ctx: RouteContext): Promise<Response>
19
44
  csv(ctx: RouteContext): Promise<Response>
45
+ /** Owner-only classification of one written answer (technical/general). */
46
+ label(ctx: RouteContext): Promise<Response>
20
47
  }
21
48
 
49
+ const SurveyAnswerLabelRequestSchema = z
50
+ .object({
51
+ responseId: BoundedRecordIdSchema,
52
+ questionId: SurveyQuestionIdSchema,
53
+ label: z.enum(['technical', 'general']),
54
+ })
55
+ .strict()
56
+
22
57
  export function createSurveyRoutes(config: SurveyRouteConfig): SurveyRouteHandlers {
23
58
  const providers: CmsProviders = { ...nullProviders, ...(config.providers ?? {}) }
24
59
 
@@ -37,7 +72,23 @@ export function createSurveyRoutes(config: SurveyRouteConfig): SurveyRouteHandle
37
72
  if (!parsed.success || parsed.data.survey.id !== surveyId) {
38
73
  return surveyErrorResponse('survey_provider_malformed')
39
74
  }
40
- return responseJson(parsed.data)
75
+ const caller = await resolveSurveyCaller(config.authz, ctx)
76
+ // Both roles share one path: any written answer on the page needs
77
+ // the label index — the owner view decorates with it, the non-owner
78
+ // view withholds by it, and auto-labeling (when enabled) schedules
79
+ // against it after the response is built. The unlabeled set is
80
+ // computed once: the scheduler dispatches (a bounded slice of) it,
81
+ // and the owner payload counts it.
82
+ if (!hasWrittenAnswers(parsed.data)) return responseJson(parsed.data)
83
+ const labels = await loadAnswerLabelsOrFail(ctx.db, surveyId)
84
+ if (labels instanceof Response) return labels
85
+ const unlabeled = unlabeledWrittenAnswers(parsed.data, labels)
86
+ scheduleAutoLabelIfEnabled(ctx, config, surveyId, unlabeled)
87
+ return responseJson(
88
+ canLabelSurveyAnswers(caller?.role)
89
+ ? withOwnerLabelCounts(withOwnerLabels(parsed.data, labels), unlabeled)
90
+ : withholdNonGeneralAnswers(parsed.data, labels),
91
+ )
41
92
  } catch (error) {
42
93
  return providerFailure(error)
43
94
  }
@@ -53,17 +104,59 @@ export function createSurveyRoutes(config: SurveyRouteConfig): SurveyRouteHandle
53
104
  await providers.surveys?.getCsv({ surveyId }),
54
105
  )
55
106
  if (!parsed.success) return surveyErrorResponse('survey_provider_malformed')
56
- return new Response(parsed.data.content, {
57
- headers: {
58
- 'Cache-Control': 'no-store',
59
- 'Content-Disposition': `attachment; filename="${safeCsvFilename(parsed.data.filename)}"`,
60
- 'Content-Type': 'text/csv; charset=utf-8',
61
- },
62
- })
107
+ const caller = await resolveSurveyCaller(config.authz, ctx)
108
+ // The owner exports byte-identical provider bytes; every other role
109
+ // gets text cells redacted before the bytes leave the Worker.
110
+ if (!canLabelSurveyAnswers(caller?.role)) {
111
+ const redacted = await redactCsvForNonOwner(providers, ctx, surveyId, parsed.data.content)
112
+ if (redacted instanceof Response) return redacted
113
+ return csvResponse({ filename: parsed.data.filename, content: redacted })
114
+ }
115
+ return csvResponse(parsed.data)
63
116
  } catch (error) {
64
117
  return providerFailure(error)
65
118
  }
66
119
  },
120
+
121
+ async label(ctx) {
122
+ const denied = await requireSurveyAccess(config.authz, ctx)
123
+ if (denied) return withNoStoreAuthFailure(denied)
124
+ const caller = await resolveSurveyCaller(config.authz, ctx)
125
+ if (!canLabelSurveyAnswers(caller?.role)) return surveyErrorResponse('survey_forbidden')
126
+ const surveyId = parseSurveyId(ctx)
127
+ if (!surveyId) return surveyErrorResponse('survey_invalid_request')
128
+ let body: unknown
129
+ try {
130
+ body = await ctx.request.json()
131
+ } catch {
132
+ return surveyErrorResponse('survey_invalid_request')
133
+ }
134
+ const parsed = SurveyAnswerLabelRequestSchema.safeParse(body)
135
+ if (!parsed.success) return surveyErrorResponse('survey_invalid_request')
136
+ try {
137
+ await ctx.db
138
+ .prepare(
139
+ `INSERT INTO survey_answer_labels
140
+ (survey_id, response_id, question_id, label, source, labeled_by, labeled_at)
141
+ VALUES (?, ?, ?, ?, 'owner', ?, ?)
142
+ ON CONFLICT(survey_id, response_id, question_id) DO UPDATE SET
143
+ label = excluded.label, source = 'owner',
144
+ labeled_by = excluded.labeled_by, labeled_at = excluded.labeled_at`,
145
+ )
146
+ .bind(
147
+ surveyId,
148
+ parsed.data.responseId,
149
+ parsed.data.questionId,
150
+ parsed.data.label,
151
+ caller?.subject ?? null,
152
+ Date.now(),
153
+ )
154
+ .run()
155
+ } catch {
156
+ return surveyErrorResponse('survey_internal_failure')
157
+ }
158
+ return responseJson({ surveyId, ...parsed.data })
159
+ },
67
160
  }
68
161
  }
69
162
 
@@ -72,6 +165,488 @@ async function requireSurveyAccess(authz: Authz, ctx: RouteContext): Promise<Res
72
165
  return authz.requireRole('view_survey_responses' satisfies Capability, ctx)
73
166
  }
74
167
 
168
+ /**
169
+ * The caller's CMS role, resolved server-side from the injected authz seam.
170
+ * A host that has not wired `getUser` gets null — which every label decision
171
+ * reads as "not the owner", so an unwired seam withholds rather than leaks.
172
+ */
173
+ async function resolveSurveyCaller(authz: Authz, ctx: RouteContext): Promise<CmsUser | null> {
174
+ if (!authz.getUser) return null
175
+ try {
176
+ return await authz.getUser(ctx)
177
+ } catch {
178
+ return null
179
+ }
180
+ }
181
+
182
+ type AnswerLabelIndex = Map<string, { label: SurveyAnswerLabel; source: string }>
183
+
184
+ function answerLabelKey(responseId: string, questionId: string): string {
185
+ return `${responseId}\u0000${questionId}`
186
+ }
187
+
188
+ async function loadAnswerLabels(db: D1Database, surveyId: string): Promise<AnswerLabelIndex> {
189
+ const result = await db
190
+ .prepare(
191
+ 'SELECT response_id, question_id, label, source FROM survey_answer_labels WHERE survey_id = ?',
192
+ )
193
+ .bind(surveyId)
194
+ .all<{ response_id: string; question_id: string; label: string; source: string }>()
195
+ const labels: AnswerLabelIndex = new Map()
196
+ for (const row of result.results ?? []) {
197
+ if (row.label === 'technical' || row.label === 'general') {
198
+ labels.set(answerLabelKey(row.response_id, row.question_id), {
199
+ label: row.label,
200
+ source: row.source,
201
+ })
202
+ }
203
+ }
204
+ return labels
205
+ }
206
+
207
+ /**
208
+ * Label-store reads fail loudly (500), never by degrading to an unfiltered
209
+ * payload: a silent empty read would hand technical comments to non-owners.
210
+ */
211
+ async function loadAnswerLabelsOrFail(
212
+ db: D1Database,
213
+ surveyId: string,
214
+ ): Promise<AnswerLabelIndex | Response> {
215
+ try {
216
+ return await loadAnswerLabels(db, surveyId)
217
+ } catch {
218
+ return surveyErrorResponse('survey_internal_failure')
219
+ }
220
+ }
221
+
222
+ type SurveyAnswer = NonNullable<SurveyResults['responses'][number]['answers']>[number]
223
+
224
+ type QuestionIndex = {
225
+ textQuestionIds: Set<string>
226
+ allQuestionIds: Set<string>
227
+ /** Declared option values per choice question — the only choice values that are not written text. */
228
+ choiceOptions: Map<string, Set<string>>
229
+ }
230
+
231
+ function questionIndex(data: SurveyResults): QuestionIndex {
232
+ const choiceOptions = new Map<string, Set<string>>()
233
+ for (const question of data.questions) {
234
+ if (question.kind === 'single_choice' || question.kind === 'multi_choice') {
235
+ choiceOptions.set(question.id, new Set(question.options))
236
+ }
237
+ }
238
+ return {
239
+ textQuestionIds: new Set(
240
+ data.questions.filter((question) => question.kind === 'text').map((question) => question.id),
241
+ ),
242
+ allQuestionIds: new Set(data.questions.map((question) => question.id)),
243
+ choiceOptions,
244
+ }
245
+ }
246
+
247
+ function emptyQuestionIndex(): QuestionIndex {
248
+ return { textQuestionIds: new Set(), allQuestionIds: new Set(), choiceOptions: new Map() }
249
+ }
250
+
251
+ function hasWrittenAnswers(data: SurveyResults): boolean {
252
+ if (data.freeText.length > 0) return true
253
+ const questions = questionIndex(data)
254
+ return data.responses.some((response) =>
255
+ (response.answers ?? []).some((answer) => isWrittenAnswer(answer, questions)),
256
+ )
257
+ }
258
+
259
+ /**
260
+ * A written (potentially technical) answer: one attached to a text question,
261
+ * a choice value the question never declared ("Other: …" free text,
262
+ * sec-review publishing #22), or a string or array answer to a question the
263
+ * payload no longer knows — a retired text question stays withheld rather
264
+ * than leaking. Declared choice selections and scale numbers are never
265
+ * written answers.
266
+ */
267
+ function isWrittenAnswer(answer: SurveyAnswer, questions: QuestionIndex): boolean {
268
+ if (questions.textQuestionIds.has(answer.questionId)) return true
269
+ const options = questions.choiceOptions.get(answer.questionId)
270
+ if (!options) return typeof answer.value !== 'number'
271
+ if (typeof answer.value === 'string') return !options.has(answer.value)
272
+ if (Array.isArray(answer.value)) return answer.value.some((value) => !options.has(value))
273
+ return false
274
+ }
275
+
276
+ /**
277
+ * The non-owner view is an allow-list, not a block-list (operator ruling
278
+ * 2026-10-02, restated in the packet addendum): only answers labeled 'general'
279
+ * pass. Technical AND unlabeled answers are withheld — a new comment is
280
+ * owner-only until it is explicitly labeled general. Declared choice
281
+ * selections and scale values are not written answers, so they are unaffected;
282
+ * a free-typed choice value is written text and is withheld.
283
+ */
284
+ function withholdNonGeneralAnswers(data: SurveyResults, labels: AnswerLabelIndex): SurveyResults {
285
+ const questions = questionIndex(data)
286
+ return {
287
+ ...data,
288
+ freeText: data.freeText.filter(
289
+ (entry) =>
290
+ labels.get(answerLabelKey(entry.responseId, entry.questionId))?.label === 'general',
291
+ ),
292
+ responses: data.responses.map((response) => ({
293
+ ...response,
294
+ answers: (response.answers ?? []).flatMap((answer) => {
295
+ const kept = nonOwnerAnswer(answer, response.id, questions, labels)
296
+ return kept === null ? [] : [kept]
297
+ }),
298
+ })),
299
+ }
300
+ }
301
+
302
+ /**
303
+ * One answer as a non-owner may see it, or null to withhold it: 'general'
304
+ * passes untouched; a wholly written answer is withheld; a multi-choice that
305
+ * mixes declared selections with free-typed values keeps only the declared
306
+ * ones, so a declared selection never carries its sibling "Other: …" out.
307
+ */
308
+ function nonOwnerAnswer(
309
+ answer: SurveyAnswer,
310
+ responseId: string,
311
+ questions: QuestionIndex,
312
+ labels: AnswerLabelIndex,
313
+ ): SurveyAnswer | null {
314
+ if (labels.get(answerLabelKey(responseId, answer.questionId))?.label === 'general') return answer
315
+ if (questions.textQuestionIds.has(answer.questionId)) return null
316
+ const options = questions.choiceOptions.get(answer.questionId)
317
+ if (!options) {
318
+ // A question the payload no longer knows: strings and arrays stay
319
+ // withheld (fail closed); numbers pass as scale-like.
320
+ return typeof answer.value === 'number' ? answer : null
321
+ }
322
+ if (typeof answer.value === 'string') return options.has(answer.value) ? answer : null
323
+ if (Array.isArray(answer.value)) {
324
+ const declared = answer.value.filter((value) => options.has(value))
325
+ if (declared.length === answer.value.length) return answer
326
+ return declared.length > 0 ? { ...answer, value: declared } : null
327
+ }
328
+ return answer
329
+ }
330
+
331
+ /**
332
+ * The owner sees everything, with each written answer's current label and its
333
+ * provenance (`labelSource`: who labeled it, so the screen can badge the
334
+ * automatic ones). Non-owner payloads are never decorated — they pass through
335
+ * withholdNonGeneralAnswers untouched.
336
+ */
337
+ function withOwnerLabels(data: SurveyResults, labels: AnswerLabelIndex): SurveyResults {
338
+ return {
339
+ ...data,
340
+ freeText: data.freeText.map((entry) => {
341
+ const row = labels.get(answerLabelKey(entry.responseId, entry.questionId))
342
+ if (row === undefined) return entry
343
+ const labelSource = isAnswerLabelSource(row.source) ? row.source : undefined
344
+ return labelSource === undefined
345
+ ? { ...entry, label: row.label }
346
+ : { ...entry, label: row.label, labelSource }
347
+ }),
348
+ }
349
+ }
350
+
351
+ function isAnswerLabelSource(value: string): value is 'owner' | 'auto' | 'import' {
352
+ return value === 'owner' || value === 'auto' || value === 'import'
353
+ }
354
+
355
+ /**
356
+ * The owner payload also says how many written answers on this page are still
357
+ * unlabeled — the whole team-hidden set, plus the subset the personal-details
358
+ * pre-filter will never dispatch (owner labels those by hand). Attached only
359
+ * when non-zero; non-owner payloads are never decorated.
360
+ */
361
+ function withOwnerLabelCounts(
362
+ data: SurveyResults,
363
+ unlabeled: SurveyAutoLabelCandidate[],
364
+ ): SurveyResults {
365
+ const personalDetails = unlabeled.filter((candidate) =>
366
+ containsPersonalDetails(candidate.answerText),
367
+ ).length
368
+ return {
369
+ ...data,
370
+ ...(unlabeled.length > 0 ? { unlabeledCount: unlabeled.length } : {}),
371
+ ...(personalDetails > 0 ? { personalDetailsCount: personalDetails } : {}),
372
+ }
373
+ }
374
+
375
+ /**
376
+ * Hand the current page's unlabeled written answers to the background
377
+ * labeler (surveys.autoLabel). A no-op unless the option is on, the host
378
+ * wired LlmDispatch and the attempt store, and the request carries a
379
+ * waitUntil seam; scheduling is synchronous, so the results response is
380
+ * already on its way.
381
+ */
382
+ function scheduleAutoLabelIfEnabled(
383
+ ctx: RouteContext,
384
+ config: SurveyRouteConfig,
385
+ surveyId: string,
386
+ unlabeled: SurveyAutoLabelCandidate[],
387
+ ): void {
388
+ if (!config.autoLabel?.llm || !ctx.waitUntil) return
389
+ const batch = unlabeled.slice(0, SURVEY_AUTO_LABEL_BATCH_LIMIT)
390
+ if (batch.length === 0) return
391
+ scheduleSurveyAutoLabel(
392
+ {
393
+ surveyId,
394
+ db: ctx.db,
395
+ llm: config.autoLabel.llm,
396
+ attempts: config.autoLabel.attempts,
397
+ waitUntil: ctx.waitUntil,
398
+ },
399
+ batch,
400
+ )
401
+ }
402
+
403
+ /**
404
+ * Every unlabeled written answer on the current page, in payload order
405
+ * (freeText first, then free-typed choice values inside responses).
406
+ * Already-labeled answers — owner or import — are never returned; the auto
407
+ * INSERT's ON CONFLICT DO NOTHING is the second guard when the in-memory
408
+ * index is stale. The scheduler caps its dispatch slice at
409
+ * SURVEY_AUTO_LABEL_BATCH_LIMIT; the owner's unlabeledCount uses the full set.
410
+ */
411
+ function unlabeledWrittenAnswers(
412
+ data: SurveyResults,
413
+ labels: AnswerLabelIndex,
414
+ ): SurveyAutoLabelCandidate[] {
415
+ if (data.freeText.length === 0 && data.responses.length === 0) return []
416
+ const questions = questionIndex(data)
417
+ const promptById = new Map(data.questions.map((question) => [question.id, question.prompt]))
418
+ const candidates: SurveyAutoLabelCandidate[] = []
419
+ const seen = new Set<string>()
420
+ const consider = (responseId: string, questionId: string, answerText: string) => {
421
+ const key = answerLabelKey(responseId, questionId)
422
+ if (seen.has(key) || labels.has(key)) return
423
+ seen.add(key)
424
+ candidates.push({
425
+ responseId,
426
+ questionId,
427
+ questionPrompt: promptById.get(questionId) ?? '',
428
+ answerText,
429
+ })
430
+ }
431
+ for (const entry of data.freeText) {
432
+ consider(entry.responseId, entry.questionId, entry.value)
433
+ }
434
+ for (const response of data.responses) {
435
+ for (const answer of response.answers ?? []) {
436
+ if (!isWrittenAnswer(answer, questions)) continue
437
+ if (typeof answer.value === 'string') {
438
+ consider(response.id, answer.questionId, answer.value)
439
+ } else if (Array.isArray(answer.value)) {
440
+ // A multi-choice mixing declared selections with free-typed values:
441
+ // only the free-typed part is written text.
442
+ const options = questions.choiceOptions.get(answer.questionId)
443
+ const written = answer.value.filter((value) => !options?.has(value))
444
+ if (written.length > 0) {
445
+ consider(response.id, answer.questionId, written.join(' '))
446
+ }
447
+ }
448
+ }
449
+ }
450
+ return candidates
451
+ }
452
+
453
+ /**
454
+ * Redacts the CSV for a non-owner: every cell that holds written text (a text
455
+ * answer, an unknown question's answer, or a choice value the question never
456
+ * declared) is blanked or reduced to its declared selections unless labeled
457
+ * 'general', and every cell beyond the header width is dropped — an overlong
458
+ * row is an unknown shape whose extras never pass through (sec-review
459
+ * publishing #22). Returns the redacted CSV text, or a Response when the shape
460
+ * is unrecognised (loud 500 — passing provider bytes through unrecognised
461
+ * would leak them) or a dependency failed. Byte-identical passthrough when
462
+ * nothing needs changing.
463
+ */
464
+ async function redactCsvForNonOwner(
465
+ providers: CmsProviders,
466
+ ctx: RouteContext,
467
+ surveyId: string,
468
+ content: string,
469
+ ): Promise<string | Response> {
470
+ const rows = parseCsvRows(content)
471
+ if (!rows || rows.length === 0) return surveyErrorResponse('survey_internal_failure')
472
+ const header = rows[0]
473
+ const answerColumns = header.reduce<number[]>((columns, column, index) => {
474
+ if (column.startsWith('answer:')) columns.push(index)
475
+ return columns
476
+ }, [])
477
+ const overlong = rows.some((row) => row.length > header.length)
478
+ if (answerColumns.length === 0 && !overlong) return content
479
+
480
+ // Which cells can hold written text: the question kinds come from the
481
+ // provider's own results payload. A column whose question is unknown is
482
+ // treated as text — an answer that cannot be proven non-text is withheld
483
+ // when it cannot be proven 'general'.
484
+ let questions = emptyQuestionIndex()
485
+ if (answerColumns.length > 0) {
486
+ try {
487
+ const results = SurveyResultsSchema.safeParse(
488
+ await providers.surveys?.getResults({ surveyId, offset: 0 }),
489
+ )
490
+ if (!results.success || results.data.survey.id !== surveyId) {
491
+ return surveyErrorResponse('survey_provider_malformed')
492
+ }
493
+ questions = questionIndex(results.data)
494
+ } catch (error) {
495
+ return providerFailure(error)
496
+ }
497
+ }
498
+
499
+ let labels: AnswerLabelIndex = new Map()
500
+ if (answerColumns.length > 0) {
501
+ const loaded = await loadAnswerLabelsOrFail(ctx.db, surveyId)
502
+ if (loaded instanceof Response) return loaded
503
+ labels = loaded
504
+ }
505
+
506
+ const responseIdIndex = header.indexOf('response_id')
507
+ let blanked = 0
508
+ for (const row of rows.slice(1)) {
509
+ if (row.length > header.length) row.length = header.length
510
+ const responseId = responseIdIndex >= 0 ? row[responseIdIndex] : null
511
+ for (const column of answerColumns) {
512
+ const questionId = header[column].slice('answer:'.length)
513
+ if (
514
+ responseId !== null &&
515
+ responseId !== '' &&
516
+ labels.get(answerLabelKey(responseId, questionId))?.label === 'general'
517
+ ) {
518
+ continue
519
+ }
520
+ const cell = row[column] ?? ''
521
+ if (cell === '' || !cellIsWritten(cell, questionId, questions)) continue
522
+ row[column] = declaredSelections(cell, questions.choiceOptions.get(questionId))
523
+ blanked += 1
524
+ }
525
+ }
526
+ if (blanked === 0 && !overlong) return content
527
+ const terminator = content.includes('\r\n') ? '\r\n' : '\n'
528
+ return `${rows.map((row) => row.map(csvField).join(',')).join(terminator)}${terminator}`
529
+ }
530
+
531
+ /** A CSV answer cell holds written text unless it is a declared choice selection. */
532
+ function cellIsWritten(cell: string, questionId: string, questions: QuestionIndex): boolean {
533
+ if (questions.textQuestionIds.has(questionId) || !questions.allQuestionIds.has(questionId)) {
534
+ return true
535
+ }
536
+ const options = questions.choiceOptions.get(questionId)
537
+ // A scale cell is an integer or empty; anything else is written text (the
538
+ // JSON path withholds a non-integer scale value the same way).
539
+ if (!options) return cell.trim() !== '' && !/^-?\d+$/.test(cell.trim())
540
+ if (options.has(cell)) return false
541
+ // A multi-choice cell serializes its selections comma-separated; the cell
542
+ // is written text unless every selection is a declared option.
543
+ return !cell.split(',').every((token) => options.has(token.trim()))
544
+ }
545
+
546
+ /** Only the declared selections of a choice cell (or '' when none or not a choice). */
547
+ function declaredSelections(cell: string, options: Set<string> | undefined): string {
548
+ if (!options) return ''
549
+ return cell
550
+ .split(',')
551
+ .filter((token) => options.has(token.trim()))
552
+ .map((token) => token.trim())
553
+ .join(', ')
554
+ }
555
+
556
+ /**
557
+ * Full RFC4180 row parse (quoted fields, escaped quotes, newlines inside
558
+ * quotes, CRLF or LF terminators). Null when the shape is unrecognised —
559
+ * an unclosed quote or junk after a closing quote. Blank lines are skipped.
560
+ */
561
+ function parseCsvRows(content: string): string[][] | null {
562
+ const source = content.startsWith('\uFEFF') ? content.slice(1) : content
563
+ const rows: string[][] = []
564
+ let row: string[] = []
565
+ let field = ''
566
+ let inQuotes = false
567
+ let closedQuote = false
568
+ let atFieldStart = true
569
+
570
+ const endField = () => {
571
+ row.push(field)
572
+ field = ''
573
+ closedQuote = false
574
+ atFieldStart = true
575
+ }
576
+ const endRow = () => {
577
+ if (row.length > 0 || field !== '') rows.push([...row, field])
578
+ row = []
579
+ field = ''
580
+ closedQuote = false
581
+ atFieldStart = true
582
+ }
583
+
584
+ for (let index = 0; index < source.length; index += 1) {
585
+ const char = source[index]
586
+ if (inQuotes) {
587
+ if (char !== '"') {
588
+ field += char
589
+ continue
590
+ }
591
+ if (source[index + 1] === '"') {
592
+ field += '"'
593
+ index += 1
594
+ continue
595
+ }
596
+ inQuotes = false
597
+ closedQuote = true
598
+ continue
599
+ }
600
+ if (closedQuote) {
601
+ if (char === ',') {
602
+ endField()
603
+ continue
604
+ }
605
+ if (char === '\n' || char === '\r') {
606
+ endRow()
607
+ if (char === '\r' && source[index + 1] === '\n') index += 1
608
+ continue
609
+ }
610
+ return null
611
+ }
612
+ if (atFieldStart && char === '"') {
613
+ inQuotes = true
614
+ atFieldStart = false
615
+ continue
616
+ }
617
+ if (char === '"') return null
618
+ if (char === ',') {
619
+ endField()
620
+ continue
621
+ }
622
+ if (char === '\n' || char === '\r') {
623
+ if (field === '' && row.length === 0) continue // blank line
624
+ endRow()
625
+ if (char === '\r' && source[index + 1] === '\n') index += 1
626
+ continue
627
+ }
628
+ field += char
629
+ atFieldStart = false
630
+ }
631
+ if (inQuotes) return null
632
+ if (field !== '' || row.length > 0) rows.push([...row, field])
633
+ return rows
634
+ }
635
+
636
+ function csvField(value: string): string {
637
+ return /[",\r\n]/.test(value) ? `"${value.replace(/"/g, '""')}"` : value
638
+ }
639
+
640
+ function csvResponse(data: { filename: string; content: string }): Response {
641
+ return new Response(data.content, {
642
+ headers: {
643
+ 'Cache-Control': 'no-store',
644
+ 'Content-Disposition': `attachment; filename="${safeCsvFilename(data.filename)}"`,
645
+ 'Content-Type': 'text/csv; charset=utf-8',
646
+ },
647
+ })
648
+ }
649
+
75
650
  function parseOffset(ctx: RouteContext): number | null {
76
651
  const raw = new URL(ctx.request.url).searchParams.get('offset')
77
652
  if (raw === null) return 0
@@ -94,7 +669,7 @@ function safeCsvFilename(value: string): string {
94
669
  const safe = basename
95
670
  .replace(/[^A-Za-z0-9._-]+/g, '-')
96
671
  .replace(/^\.+/, '')
97
- .replace(/-+/g, '-')
672
+ .replace('-+', '-')
98
673
  .slice(0, 180)
99
674
  return safe.endsWith('.csv') ? safe : `${safe || 'survey-responses'}.csv`
100
675
  }
@@ -921,6 +921,21 @@ ALTER TABLE content_insight_runs ADD COLUMN publication_run_id TEXT;
921
921
  ALTER TABLE content_insight_runs ADD COLUMN payload_sha256 TEXT;
922
922
  `,
923
923
  },
924
+ {
925
+ id: '0033_survey_answer_labels',
926
+ sql: `
927
+ CREATE TABLE survey_answer_labels (
928
+ survey_id TEXT NOT NULL,
929
+ response_id TEXT NOT NULL,
930
+ question_id TEXT NOT NULL,
931
+ label TEXT NOT NULL CHECK (label IN ('technical','general')),
932
+ source TEXT NOT NULL CHECK (source IN ('owner','auto','import')),
933
+ labeled_by TEXT,
934
+ labeled_at INTEGER NOT NULL,
935
+ PRIMARY KEY (survey_id, response_id, question_id)
936
+ );
937
+ `,
938
+ },
924
939
  ]
925
940
 
926
941
  /**