iterate 0.2.7 → 0.3.0

package/dist/starter-apps/media/worker.mjs

@@ -0,0 +1,579 @@
+import { r as StreamProcessorDurableObject } from "../../sdk-DMB-IM11.mjs";
+import { h as defineProcessorContract, s as StreamProcessor } from "../../stream-processor-keepalive-DAQTP6m3.mjs";
+import { n as isIdempotencyConflict } from "../../idempotency-DleloJNt.mjs";
+import "../../processors.mjs";
+import { t as _usingCtx } from "../../usingCtx-inzbY1Qz.mjs";
+import { mediaStreamPath } from "./ref.mjs";
+import { z } from "zod";
+//#region src/starter-apps/media/analysis.ts
+/** Sees the pixels: transcribes text verbatim and picks tags. */
+const MEDIA_VISION_MODEL = "@cf/meta/llama-4-scout-17b-16e-instruct";
+/**
+* Starter taxonomy — expect churn. Multi-tag with overlap allowed; the model
+* may also coin up to two novel kebab-case tags, and returning NO tags is an
+* acceptable answer (deliberately conservative — early dogfood tagged
+* everything `bug-report` on wild guesses).
+*/
+const MEDIA_TAG_TAXONOMY = [
+	{
+		tag: "screenshot",
+		hint: "a captured device screen: app UI, web page, status bar visible"
+	},
+	{
+		tag: "photo",
+		hint: "a camera photograph of the physical world"
+	},
+	{
+		tag: "transient",
+		hint: "one-off info with no lasting value: OTP codes, confirmations"
+	},
+	{
+		tag: "clipping",
+		hint: "a saved post, article, meme, or quote worth keeping"
+	},
+	{
+		tag: "logistics",
+		hint: "tickets, bookings, schedules, travel details, QR codes"
+	},
+	{
+		tag: "receipt",
+		hint: "proof of purchase, invoices, order confirmations"
+	},
+	{
+		tag: "conversation",
+		hint: "chat threads, DMs, emails, comment sections"
+	},
+	{
+		tag: "code",
+		hint: "code, terminals, dev tools, technical docs"
+	},
+	{
+		tag: "reference",
+		hint: "info to keep long-term: settings, lists, instructions"
+	}
+];
+/**
+* One full analysis of a stored media file. Throws on conversion failure so
+* the caller (the processor's obligation attempt) owns retry/settlement; an
+* unparseable vision answer degrades to empty title/transcript and
+* ["untagged"] so failures stay visible instead of hiding an item.
+*/
+async function analyzeMediaImage(session, input) {
+	const bytes = await session.files.get(input.path).bytes();
+	const described = await session.ai.toMarkdown({
+		name: input.filename,
+		blob: bytes
+	});
+	if (described.format === "error") throw new Error(`toMarkdown failed for ${input.filename}: ${described.error}`);
+	const markdown = (described.data || "").trim();
+	let visionBytes = bytes;
+	let visionType = input.contentType;
+	if (bytes.length > 1e6) try {
+		const resized = await session.integrations.cf.images.transformBytes({
+			image: bytes,
+			transforms: [{ width: 1280 }],
+			output: {
+				format: "image/jpeg",
+				quality: 80
+			}
+		});
+		visionBytes = resized.bytes;
+		visionType = resized.contentType;
+	} catch {}
+	let binary = "";
+	for (let i = 0; i < visionBytes.length; i += 32768) binary += String.fromCharCode(...visionBytes.subarray(i, i + 32768));
+	return {
+		...parseVisionAnswer(visionAnswerText(await session.ai.run(MEDIA_VISION_MODEL, {
+			messages: [{
+				role: "user",
+				content: [{
+					type: "text",
+					text: mediaVisionPrompt()
+				}, {
+					type: "image_url",
+					image_url: { url: `data:${visionType};base64,${btoa(binary)}` }
+				}]
+			}],
+			max_tokens: 1024
+		}))),
+		markdown
+	};
+}
+/** Both Workers AI answer shapes: `{ response }` and OpenAI-style choices. */
+function visionAnswerText(answer) {
+	const bag = answer;
+	if (typeof bag?.response === "string") return bag.response;
+	const content = bag?.choices?.[0]?.message?.content;
+	return typeof content === "string" ? content : "";
+}
+/** Extract the JSON object from a chatty model answer, defensively. */
+function parseVisionAnswer(text) {
+	let title = "";
+	let transcript = "";
+	let tags = ["untagged"];
+	const match = text.match(/\{[\s\S]*\}/);
+	if (match) try {
+		const parsed = JSON.parse(match[0]);
+		if (typeof parsed.title === "string") title = parsed.title.trim().slice(0, 120);
+		if (typeof parsed.transcript === "string") transcript = parsed.transcript.trim();
+		if (Array.isArray(parsed.tags)) tags = [...new Set(parsed.tags.filter((tag) => typeof tag === "string").map((tag) => tag.toLowerCase().trim().replace(/[^a-z0-9-]+/g, "-").replace(/^-+|-+$/g, "")).filter((tag) => tag.length > 0 && tag.length <= 30))].slice(0, 6);
+	} catch {}
+	return {
+		title,
+		transcript,
+		tags,
+		processedBy: MEDIA_VISION_MODEL
+	};
+}
+function mediaVisionPrompt() {
+	return [
+		"Reply with ONLY a JSON object: {\"title\": string, \"transcript\": string, \"tags\": string[]}.",
+		"title: ONE line saying what the image IS, specific not generic — 'Trenitalia ticket Rome→Florence 09:45', never 'Screenshot' or 'Image Description'.",
+		"transcript: ALL text visible in the image, verbatim, reading order. Empty string if there is none.",
+		"tags: pick from the list below. Include a tag ONLY when the image clearly shows it — no guesses.",
+		"Fewer tags is better; an empty array is a fine answer. Overlap is allowed.",
+		...MEDIA_TAG_TAXONOMY.map(({ tag, hint }) => `- "${tag}": ${hint}`),
+		"You may add at most 2 extra kebab-case tags if something important has no tag above."
+	].join("\n");
+}
+//#endregion
+//#region src/starter-apps/media/processor.ts
+/**
+* An analysis obligation is abandoned (settled as failed, without dialing
+* the AI) when its requesting event is older than this — the staleness
+* doctrine's horizon. Generous on purpose: a phone can upload while the
+* analyzer is broken and still get analyzed when the fix deploys; Re-analyze
+* covers anything older.
+*/
+const MEDIA_ANALYSIS_EXPIRY_MS = 1440 * 60 * 1e3;
+/** Tries per attempt, with backoff between them — covers the transient
+* Workers AI failure class ("8005: Internal server error") observed in prod
+* without an extra durable event per retry. */
+const ANALYSIS_TRIES = 3;
+const ANALYSIS_RETRY_BACKOFF_MS = [2e3, 8e3];
+/**
+* Attempts started per caught-up pass, matching the phone's old 3-wide
+* pipeline. A 19-item Delete-all re-sync once fired 19 parallel vision
+* pipelines from one pass and Workers AI answered with 8004s and 60s
+* binding timeouts — the in-attempt retries all landed inside the same
+* overloaded burst. The rest of the queue needs no timer: every settlement
+* append comes back through this processor's own delivery and the next
+* caught-up pass starts the next wave.
+*/
+const MAX_CONCURRENT_ANALYSES = 3;
+const processingFields = {
+	title: z.string().default("").meta({ description: "One-line description of what the image IS (pre-title events fold to empty)." }),
+	markdown: z.string().meta({ description: "Vision-model description — half the search corpus." }),
+	transcript: z.string().meta({ description: "Verbatim OCR of visible text — the other half." }),
+	tags: z.array(z.string()).meta({ description: "Conservative multi-label tags." }),
+	processedBy: z.string().meta({ description: "Model id that produced this processing." })
+};
+const fileFields = {
+	stableKey: z.string().meta({ description: "Content hash — the item's identity." }),
+	path: z.string().meta({ description: "itx.files path holding the bytes." }),
+	filename: z.string().meta({ description: "Original filename as picked/synced." }),
+	contentType: z.string(),
+	width: z.number(),
+	height: z.number(),
+	source: z.string().default("picker").meta({ description: "\"picker\", \"library-sync\", or \"note\" (a note attachment)." }),
+	capturedAt: z.string().nullable().default(null).meta({ description: "Asset creation time (ISO) when the source knew it." }),
+	isScreenshot: z.boolean().nullable().default(null).meta({ description: "iOS mediaSubtypes screenshot flag." })
+};
+const MediaItem = z.object({
+	...fileFields,
+	capturedEventAt: z.string().meta({ description: "Stream time the capture committed." }),
+	...processingFields,
+	analysisError: z.string().nullable().default(null).meta({ description: "The latest analysis settlement's failure, or null. Rows born from media/uploaded exist regardless of analysis outcome; a terminal failure lands here." }),
+	settledRequestOffset: z.number().nullable().default(null).meta({ description: "requestOffset of the last APPLIED settlement — settlements apply monotonically, so a late one answering an older request can never overwrite a newer result (null until a requestOffset-carrying settlement lands; legacy settlements don't advance it)." })
+});
+const PendingAnalysis = z.object({
+	stableKey: z.string(),
+	path: z.string(),
+	filename: z.string(),
+	contentType: z.string(),
+	requestOffset: z.number().meta({ description: "Offset of the uploaded/reanalyze event that opened this obligation — the settlement's idempotency identity." }),
+	expiresAtMs: z.number().meta({ description: "Epoch ms (request createdAt + MEDIA_ANALYSIS_EXPIRY_MS); past it the obligation settles as expired without dialing the AI." })
+});
+const MediaProcessorContract = defineProcessorContract({
+	slug: "media",
+	version: "0.2.1",
+	description: "Reduces captured media (screenshots/photos) on /media into a searchable index, and drives server-side vision analysis of uploaded items (obligation pattern).",
+	stateSchema: z.object({
+		items: z.record(z.string(), MediaItem).default({}).meta({ description: "Items by stableKey, latest processing overlaid." }),
+		pendingAnalyses: z.record(z.string(), PendingAnalysis).default({}).meta({ description: "Open analysis obligations by stableKey (latest request wins); settled by one media/processed event each." })
+	}),
+	events: {
+		"events.iterate.com/media/uploaded": {
+			description: "A media item's bytes landed in project files: the phone appends this cheap durable fact right after files.put (metadata only — no analysis yet), idempotency-keyed by content hash + wipe generation, sharing the media/captured key scheme so an item already captured never re-uploads as a duplicate. Opens an analysis obligation.",
+			payloadSchema: z.object(fileFields),
+			examples: [{
+				description: "A synced screenshot, uploaded and awaiting analysis.",
+				payload: {
+					stableKey: "abc123",
+					path: "/media/abc123-IMG_0001.png",
+					filename: "IMG_0001.png",
+					contentType: "image/png",
+					width: 1170,
+					height: 2532,
+					source: "library-sync",
+					capturedAt: "2026-08-10T09:00:00.000Z",
+					isScreenshot: true
+				}
+			}]
+		},
+		"events.iterate.com/media/reanalyze-requested": {
+			description: "Re-analyze an existing item (the app's Re-analyze button): opens an analysis obligation for it; the settlement overlays the item's processing fields.",
+			payloadSchema: z.object({ stableKey: z.string() }),
+			examples: [{
+				description: "A user-requested re-run.",
+				payload: { stableKey: "abc123" }
+			}]
+		},
+		"events.iterate.com/media/captured": {
+			description: "LEGACY (pre media/uploaded): a media item entered the project with its first vision processing inline, appended by the old phone-driven capture script. Still folded so history keeps rendering; nothing appends it anymore.",
+			payloadSchema: z.object({
+				...fileFields,
+				...processingFields
+			}),
+			examples: [{
+				description: "A synced screenshot with its first processing.",
+				payload: {
+					stableKey: "abc123",
+					path: "/media/abc123-IMG_0001.png",
+					filename: "IMG_0001.png",
+					contentType: "image/png",
+					width: 1170,
+					height: 2532,
+					source: "library-sync",
+					capturedAt: "2026-08-10T09:00:00.000Z",
+					isScreenshot: true,
+					markdown: "A train ticket from Rome to Florence.",
+					transcript: "Trenitalia 09:45",
+					tags: ["screenshot", "logistics"],
+					processedBy: "@cf/meta/llama-4-scout-17b-16e-instruct"
+				}
+			}]
+		},
+		"events.iterate.com/media/wiped": {
+			description: "Everything before this tombstone is gone: the wipe script deleted the stored files and every derivation (phone list, this fold) resets. Deliberate per-invocation idempotency key.",
+			payloadSchema: z.object({
+				deletedFiles: z.number().meta({ description: "Files actually deleted." }),
+				items: z.number().meta({ description: "Items that existed at wipe time." })
+			}),
+			examples: [{
+				description: "A from-scratch reset.",
+				payload: {
+					deletedFiles: 24,
+					items: 24
+				}
+			}]
+		},
+		"events.iterate.com/media/processed": {
+			description: "One analysis settlement (the obligation's single terminal event): on success the processing fields overlay the item's, `error: null`; on terminal failure `error` says why and the item's previous fields stay. Also appended by legacy phone-driven re-analysis (no error/requestOffset).",
+			payloadSchema: z.object({
+				stableKey: z.string(),
+				...processingFields,
+				error: z.string().nullable().default(null).meta({ description: "Terminal analysis failure, or null on success." }),
+				requestOffset: z.number().nullable().default(null).meta({ description: "The uploaded/reanalyze event this settles (null on legacy appends)." })
+			}),
+			examples: [{
+				description: "A successful server-side analysis of an uploaded item.",
+				payload: {
+					stableKey: "abc123",
+					title: "Trenitalia ticket Rome→Florence 09:45",
+					markdown: "A train ticket from Rome to Florence.",
+					transcript: "Trenitalia 09:45",
+					tags: ["screenshot", "logistics"],
+					processedBy: "@cf/meta/llama-4-scout-17b-16e-instruct",
+					error: null,
+					requestOffset: 12
+				}
+			}]
+		}
+	},
+	consumes: [
+		"events.iterate.com/media/uploaded",
+		"events.iterate.com/media/reanalyze-requested",
+		"events.iterate.com/media/captured",
+		"events.iterate.com/media/processed",
+		"events.iterate.com/media/wiped"
+	],
+	emits: ["events.iterate.com/media/processed"]
+});
+var MediaProcessor = class extends StreamProcessor {
+	contract = MediaProcessorContract;
+	/**
+	* Runtime-only attempts by stableKey. The uploaded/reanalyze +
+	* processed events are the durable truth: after an eviction this set is
+	* empty and the next caught-up pass restarts any still-open obligation.
+	*/
+	#liveAnalyses = /* @__PURE__ */ new Set();
+	reduce({ event, state }) {
+		switch (event.type) {
+			case "events.iterate.com/media/uploaded":
+				if (state.items[event.payload.stableKey] !== void 0) return state;
+				return {
+					...state,
+					items: {
+						...state.items,
+						[event.payload.stableKey]: {
+							...event.payload,
+							capturedEventAt: event.createdAt,
+							title: "",
+							markdown: "",
+							transcript: "",
+							tags: [],
+							processedBy: "",
+							analysisError: null,
+							settledRequestOffset: null
+						}
+					},
+					pendingAnalyses: {
+						...state.pendingAnalyses,
+						[event.payload.stableKey]: pendingAnalysisFor(event)
+					}
+				};
+			case "events.iterate.com/media/reanalyze-requested": {
+				const item = state.items[event.payload.stableKey];
+				if (item === void 0) return state;
+				return {
+					...state,
+					pendingAnalyses: {
+						...state.pendingAnalyses,
+						[item.stableKey]: {
+							stableKey: item.stableKey,
+							path: item.path,
+							filename: item.filename,
+							contentType: item.contentType,
+							requestOffset: event.offset,
+							expiresAtMs: Date.parse(event.createdAt) + MEDIA_ANALYSIS_EXPIRY_MS
+						}
+					}
+				};
+			}
+			case "events.iterate.com/media/captured":
+				if (state.items[event.payload.stableKey] !== void 0) return state;
+				return {
+					...state,
+					items: {
+						...state.items,
+						[event.payload.stableKey]: {
+							...event.payload,
+							capturedEventAt: event.createdAt,
+							analysisError: null,
+							settledRequestOffset: null
+						}
+					}
+				};
+			case "events.iterate.com/media/wiped": return {
+				...state,
+				items: {},
+				pendingAnalyses: {}
+			};
+			case "events.iterate.com/media/processed": {
+				const pendingEntry = state.pendingAnalyses[event.payload.stableKey];
+				const existing = state.items[event.payload.stableKey];
+				const requestOffset = event.payload.requestOffset === void 0 ? null : event.payload.requestOffset;
+				const settledFloor = Math.max(pendingEntry === void 0 ? 0 : pendingEntry.requestOffset, existing?.settledRequestOffset || 0);
+				if (requestOffset !== null && requestOffset < settledFloor) return state;
+				const { [event.payload.stableKey]: _settled, ...pendingAnalyses } = state.pendingAnalyses;
+				if (existing === void 0) return {
+					...state,
+					pendingAnalyses
+				};
+				const error = event.payload.error || null;
+				const settledRequestOffset = requestOffset === null ? existing.settledRequestOffset : requestOffset;
+				const item = error === null ? {
+					...existing,
+					title: event.payload.title,
+					markdown: event.payload.markdown,
+					transcript: event.payload.transcript,
+					tags: event.payload.tags,
+					processedBy: event.payload.processedBy,
+					analysisError: null,
+					settledRequestOffset
+				} : {
+					...existing,
+					analysisError: error,
+					settledRequestOffset
+				};
+				return {
+					...state,
+					items: {
+						...state.items,
+						[event.payload.stableKey]: item
+					},
+					pendingAnalyses
+				};
+			}
+			default: return state;
+		}
+	}
+	processEvent(args) {
+		if (!args.delivery.caughtUp) return;
+		const now = this.deps.now();
+		const expired = [];
+		for (const pending of Object.values(args.state.pendingAnalyses)) {
+			if (this.#liveAnalyses.has(pending.stableKey)) continue;
+			if (now >= pending.expiresAtMs) {
+				expired.push(pending);
+				continue;
+			}
+			if (this.#liveAnalyses.size >= MAX_CONCURRENT_ANALYSES) continue;
+			this.#liveAnalyses.add(pending.stableKey);
+			args.runInBackground(async () => {
+				try {
+					const settlement = await this.#attemptAnalysis(pending);
+					await this.#appendSettlement(args.append, pending, settlement);
+				} finally {
+					this.#liveAnalyses.delete(pending.stableKey);
+				}
+			});
+		}
+		if (expired.length === 0) return;
+		args.blockProcessorWhile(async () => {
+			for (const pending of expired) await this.#appendSettlement(args.append, pending, { error: `analysis window expired before it could run (older than ${MEDIA_ANALYSIS_EXPIRY_MS / 36e5}h) — use Re-analyze to retry` });
+		});
+	}
+	/** One attempt = up to {@link ANALYSIS_TRIES} tries with short backoff. */
+	async #attemptAnalysis(pending) {
+		let lastError = "analysis failed";
+		for (let tryIndex = 0; tryIndex < ANALYSIS_TRIES; tryIndex++) {
+			if (tryIndex > 0) await this.deps.sleep(ANALYSIS_RETRY_BACKOFF_MS[Math.min(tryIndex - 1, ANALYSIS_RETRY_BACKOFF_MS.length - 1)]);
+			try {
+				return { result: await this.deps.analyze(pending) };
+			} catch (error) {
+				lastError = error instanceof Error ? error.message : String(error);
+			}
+		}
+		return { error: lastError.slice(0, 2e3) };
+	}
+	/**
+	* The obligation's single terminal event. The key is state-derived and
+	* deterministic per obligation (`@<stableKey>:<requestOffset>`); bodies
+	* contain AI output, so two racing incarnations can collide
+	* same-key/different-body — the first committed settlement is the
+	* authority and the loser tolerates the conflict (ai-linter shape).
+	*/
+	async #appendSettlement(append, pending, outcome) {
+		const event = {
+			type: "events.iterate.com/media/processed",
+			idempotencyKey: this.idempotencyKey(`analysis-settled@${pending.stableKey}:${pending.requestOffset}`),
+			payload: {
+				stableKey: pending.stableKey,
+				requestOffset: pending.requestOffset,
+				..."result" in outcome ? {
+					...outcome.result,
+					error: null
+				} : {
+					title: "",
+					markdown: "",
+					transcript: "",
+					tags: [],
+					processedBy: "",
+					error: outcome.error
+				}
+			}
+		};
+		try {
+			await append(event);
+		} catch (error) {
+			if (!isIdempotencyConflict(error)) throw error;
+		}
+	}
+};
+function pendingAnalysisFor(event) {
+	return {
+		stableKey: event.payload.stableKey,
+		path: event.payload.path,
+		filename: event.payload.filename,
+		contentType: event.payload.contentType,
+		requestOffset: event.offset,
+		expiresAtMs: Date.parse(event.createdAt) + MEDIA_ANALYSIS_EXPIRY_MS
+	};
+}
+/**
+* Search semantics shared with the phone's client-side filter
+* (apps/mobile/src/lib/media.ts filterMedia — kept in sync by hand, the two
+* runtimes can't share a module yet): every whitespace-separated term must
+* appear in description, transcript, filename, stored name, or tags; every
+* selected tag must be present. Newest first.
+*/
+function searchMediaItems(state, query) {
+	const terms = (query.q || "").toLowerCase().split(/\s+/).filter(Boolean);
+	const requiredTags = query.tags || [];
+	return Object.values(state.items).filter((item) => {
+		const storedName = (item.path.split("/").at(-1) || "").replace(/^[0-9a-f]{32,}-/, "");
+		const haystack = `${item.title} ${item.markdown} ${item.transcript} ${item.filename} ${storedName} ${item.tags.join(" ")}`.toLowerCase();
+		return terms.every((term) => haystack.includes(term)) && requiredTags.every((tag) => item.tags.includes(tag));
+	}).sort((a, b) => (b.capturedAt || b.capturedEventAt).localeCompare(a.capturedAt || a.capturedEventAt)).slice(0, query.limit || 20);
+}
+//#endregion
+//#region src/starter-apps/media/worker.ts
+var MediaApp = class extends StreamProcessorDurableObject {
+	streamPath = mediaStreamPath;
+	/** Analysis obligations must survive eviction: the keepalive revives this
+	* DO and the caught-up pass restarts still-open work from reduced state. */
+	recovery = true;
+	createProcessor(deps) {
+		return new MediaProcessor({
+			...deps,
+			now: Date.now,
+			sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
+			analyze: async (input) => {
+				try {
+					var _usingCtx$1 = _usingCtx();
+					return await analyzeMediaImage(_usingCtx$1.u(await this.env.ITX.get()), input);
+				} catch (_) {
+					_usingCtx$1.e = _;
+				} finally {
+					_usingCtx$1.d();
+				}
+			}
+		});
+	}
+	/** Project-worker event delivery calls this after a durable /media event
+	* commits. Catch-up owns validation, ordering, checkpointing, and dedupe. */
+	async syncEvent(event) {
+		if (event.path !== "/media") return;
+		const registry = await this.registry();
+		await registry.catchUp("media");
+		registry.refreshLive();
+	}
+	/**
+	* Keyword search over descriptions, OCR transcripts, filenames, and tags —
+	* "train ticket" finds the train ticket because the vision model's prose
+	* says so. `tags` all must be present; newest first; default limit 20.
+	*/
+	async search(query) {
+		try {
+			var _usingCtx3 = _usingCtx();
+			await (await this.registry()).catchUp("media");
+			const { state } = await this.snapshot();
+			const hits = searchMediaItems(state, query);
+			const project = _usingCtx3.u(await this.env.ITX.get());
+			return await Promise.all(hits.map(async (item) => ({
+				...item,
+				url: await project.files.get(item.path).url()
+			})));
+		} catch (_) {
+			_usingCtx3.e = _;
+		} finally {
+			_usingCtx3.d();
+		}
+	}
+	/** Newest items, no filtering — `search` with an empty query. */
+	async list(query) {
+		return this.search({ limit: query.limit });
+	}
+	/** One item by its content hash, or null. */
+	async get(stableKey) {
+		await (await this.registry()).catchUp("media");
+		const { state } = await this.snapshot();
+		return state.items[stableKey] || null;
+	}
+};
+//#endregion
+export { MediaApp };
+
+//# sourceMappingURL=worker.mjs.map