@pithy-sh/media 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +17 -0
- package/package.json +52 -0
- package/pithy.manifest.json +85 -0
- package/src/ai/enrich.ts +187 -0
- package/src/ai/videoBatching.ts +130 -0
- package/src/capability.ts +129 -0
- package/src/cloudflare-test.d.ts +15 -0
- package/src/config/config.ts +155 -0
- package/src/data/enums.ts +53 -0
- package/src/data/extend.ts +135 -0
- package/src/data/mediaAsset.ts +78 -0
- package/src/data/mediaHash.ts +24 -0
- package/src/data/tables.ts +45 -0
- package/src/deliver/url.ts +78 -0
- package/src/error/errors.ts +95 -0
- package/src/hash/duplicates.ts +129 -0
- package/src/hash/sha256.ts +34 -0
- package/src/http/dispatch.ts +86 -0
- package/src/http/guard.ts +25 -0
- package/src/http/handlers.ts +205 -0
- package/src/http/routes.ts +119 -0
- package/src/http/schemas.ts +90 -0
- package/src/index.ts +39 -0
- package/src/migrations/0001_init.ts +104 -0
- package/src/migrations/extend.ts +40 -0
- package/src/provision/provisionMedia.ts +226 -0
- package/src/provision/resolveMediaConfig.ts +78 -0
- package/src/record/d1Store.ts +95 -0
- package/src/record/hashStore.ts +104 -0
- package/src/record/kvStore.ts +147 -0
- package/src/record/resolve.ts +34 -0
- package/src/record/store.ts +50 -0
- package/src/secret/registry.ts +85 -0
- package/src/storage/backend.ts +29 -0
- package/src/storage/cloudflare.ts +77 -0
- package/src/storage/minter.ts +57 -0
- package/src/storage/resolve.ts +78 -0
- package/src/storage/storage.ts +130 -0
- package/src/version.generated.ts +16 -0
- package/src/workflows/enrich.ts +116 -0
- package/src/workflows/hls.ts +99 -0
- package/src/workflows/retryPolicy.ts +56 -0
- package/src/workflows/specs.ts +75 -0
- package/src/workflows/worker.ts +192 -0
- package/src/workflows/wrangler.jsonc +73 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Pithy
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# @pithy-sh/media
|
|
2
|
+
|
|
3
|
+
Store, track, and enrich media on your own Cloudflare account. Images, video, audio, documents. Config picks the backend; the package does the rest.
|
|
4
|
+
|
|
5
|
+
Bytes never proxy through your Worker — the client uploads straight to Cloudflare.
|
|
6
|
+
|
|
7
|
+
```sh
|
|
8
|
+
pithy add media
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
**Documentation: [pithy.sh/docs/capabilities/media](https://pithy.sh/docs/capabilities/media).** Overview, adding it, using it, and the reference: the backends, provisioning, AI enrichment.
|
|
12
|
+
|
|
13
|
+
_Everything else is on the site. `pithy.sh/docs` is canonical — new prose goes there, not here._
|
|
14
|
+
|
|
15
|
+
## License
|
|
16
|
+
|
|
17
|
+
MIT — adopter-side app value. The root `LICENSE` covers it.
|
package/package.json
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@pithy-sh/media",
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"license": "MIT",
|
|
5
|
+
"repository": {
|
|
6
|
+
"type": "git",
|
|
7
|
+
"url": "git+https://github.com/pithy-sh/pithy.git",
|
|
8
|
+
"directory": "packages/media"
|
|
9
|
+
},
|
|
10
|
+
"files": [
|
|
11
|
+
"src",
|
|
12
|
+
"pithy.manifest.json",
|
|
13
|
+
"!src/**/*.test.*"
|
|
14
|
+
],
|
|
15
|
+
"type": "module",
|
|
16
|
+
"engines": {
|
|
17
|
+
"node": ">=22"
|
|
18
|
+
},
|
|
19
|
+
"exports": {
|
|
20
|
+
"./src/*": "./src/*.ts"
|
|
21
|
+
},
|
|
22
|
+
"scripts": {
|
|
23
|
+
"build": "tsc -p tsconfig.json --noEmit false --outDir dist",
|
|
24
|
+
"typecheck": "tsc -p tsconfig.json",
|
|
25
|
+
"test": "vitest run",
|
|
26
|
+
"test:node": "vitest run --project=node",
|
|
27
|
+
"test:workers": "vitest run --project=workers",
|
|
28
|
+
"clean": "rm -rf dist .turbo",
|
|
29
|
+
"reset": "bun run clean && rm -rf node_modules"
|
|
30
|
+
},
|
|
31
|
+
"dependencies": {
|
|
32
|
+
"@cloudflare/workers-types": "^5.20260729.1",
|
|
33
|
+
"@hono/zod-validator": "^0.9.0",
|
|
34
|
+
"@pithy-sh/cloudflare": "workspace:*",
|
|
35
|
+
"@pithy-sh/core": "workspace:*",
|
|
36
|
+
"@pithy-sh/secrets": "workspace:*",
|
|
37
|
+
"@pithy-sh/storage": "workspace:*",
|
|
38
|
+
"hono": "^4.13.2",
|
|
39
|
+
"kysely": "^0.29.0",
|
|
40
|
+
"zod": "^4.0.0"
|
|
41
|
+
},
|
|
42
|
+
"devDependencies": {
|
|
43
|
+
"@cloudflare/vitest-plugin": "^1.0.0",
|
|
44
|
+
"@pithy-sh/tsconfig": "workspace:*",
|
|
45
|
+
"@types/node": "^22.15.0",
|
|
46
|
+
"@vitest/coverage-v8": "^4.1.0",
|
|
47
|
+
"kysely-d1": "^0.4.0",
|
|
48
|
+
"typescript": "^7.0.2",
|
|
49
|
+
"vitest": "^4.1.0",
|
|
50
|
+
"wrangler": "^4.115.0"
|
|
51
|
+
}
|
|
52
|
+
}
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "media",
|
|
3
|
+
"package": "@pithy-sh/media",
|
|
4
|
+
"requiredBindings": [
|
|
5
|
+
{ "type": "d1", "name": "DB" },
|
|
6
|
+
{ "type": "r2", "name": "MEDIA_BUCKET" },
|
|
7
|
+
{
|
|
8
|
+
"type": "workflow",
|
|
9
|
+
"name": "MEDIA_IMAGE_TO_TEXT",
|
|
10
|
+
"job": "image-to-text",
|
|
11
|
+
"className": "MediaImageToTextWorkflow",
|
|
12
|
+
"optional": true
|
|
13
|
+
},
|
|
14
|
+
{
|
|
15
|
+
"type": "workflow",
|
|
16
|
+
"name": "MEDIA_AUDIO_TRANSCRIBE",
|
|
17
|
+
"job": "audio-transcribe",
|
|
18
|
+
"className": "MediaAudioTranscribeWorkflow",
|
|
19
|
+
"optional": true
|
|
20
|
+
},
|
|
21
|
+
{
|
|
22
|
+
"type": "workflow",
|
|
23
|
+
"name": "MEDIA_VIDEO_TRANSCRIBE",
|
|
24
|
+
"job": "video-transcribe",
|
|
25
|
+
"className": "MediaVideoTranscribeWorkflow",
|
|
26
|
+
"optional": true
|
|
27
|
+
},
|
|
28
|
+
{
|
|
29
|
+
"type": "workflow",
|
|
30
|
+
"name": "MEDIA_DOC_EXTRACT",
|
|
31
|
+
"job": "doc-extract",
|
|
32
|
+
"className": "MediaDocExtractWorkflow",
|
|
33
|
+
"optional": true
|
|
34
|
+
}
|
|
35
|
+
],
|
|
36
|
+
"peerCapabilities": ["secrets"],
|
|
37
|
+
"optionalCapabilities": ["auth"],
|
|
38
|
+
"migrationNamespace": "media",
|
|
39
|
+
"secrets": [
|
|
40
|
+
{
|
|
41
|
+
"name": "media-storage-credentials",
|
|
42
|
+
"origin": {
|
|
43
|
+
"kind": "obtained",
|
|
44
|
+
"issuer": "cloudflare",
|
|
45
|
+
"documentation": "https://developers.cloudflare.com/fundamentals/api/get-started/create-token/"
|
|
46
|
+
},
|
|
47
|
+
"rotation": {
|
|
48
|
+
"kind": "provider",
|
|
49
|
+
"issuer": "cloudflare",
|
|
50
|
+
"documentation": "https://developers.cloudflare.com/api/resources/user/subresources/tokens/methods/update/"
|
|
51
|
+
}
|
|
52
|
+
},
|
|
53
|
+
{
|
|
54
|
+
"name": "media-r2-credentials",
|
|
55
|
+
"origin": {
|
|
56
|
+
"kind": "obtained",
|
|
57
|
+
"issuer": "cloudflare",
|
|
58
|
+
"documentation": "https://developers.cloudflare.com/r2/api/tokens/"
|
|
59
|
+
},
|
|
60
|
+
"rotation": {
|
|
61
|
+
"kind": "manual",
|
|
62
|
+
"issuer": "cloudflare",
|
|
63
|
+
"documentation": "https://developers.cloudflare.com/r2/api/tokens/"
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
],
|
|
67
|
+
"whenToEnable": "Store, track, and enrich media — images (R2 or Cloudflare Images), video (R2 or Cloudflare Stream), audio and documents (R2). Uploads use direct-upload URLs, so bytes never proxy through your Worker. Opt in per type to AI alt text, transcription, and text extraction. Records live in D1 by default (derived text is queryable) or KV — but duplicate detection always uses D1, so the DB binding is required in both modes. R2 presigning runs through @pithy-sh/storage's ObjectStore seam pointed at MEDIA_BUCKET, so media owns no bucket mechanics and none of storage's tables or routes come with it. Storage credentials are read through @pithy-sh/secrets, so the secrets capability is required; the routes need an identity, so add auth too.",
|
|
68
|
+
"scaffold": [
|
|
69
|
+
"Ensure the secrets capability is provisioned (`pithy add secrets` then `pithy secrets provision`) — the CF API token and R2 credentials are stored and read through @pithy-sh/secrets.",
|
|
70
|
+
"Create an R2 S3 access-key pair (R2 → Manage API tokens) and an API token with Images and Stream permissions. Cloudflare exposes no API for minting an R2 key pair, so this pair is yours to create — pithy stores and rotates it, but cannot mint it.",
|
|
71
|
+
"Run `pithy media provision` — it creates the R2 bucket (and the `MEDIA` KV namespace when `recordStore: 'kv'`), writes two per-environment secrets, and deploys the media enrichment worker hosting the four Workflows for each environment. `media-storage-credentials` holds your Images + Stream API token; `media-r2-credentials` holds the R2 key pair and that environment's bucket, and is read by @pithy-sh/storage's ObjectStore, which media presigns through. Pass the credentials with `--api-token`, `--r2-access-key-id`, and `--r2-secret-access-key`, or set `R2_CREDENTIALS` in `.dev.vars`. Every step is idempotent; re-run it freely.",
|
|
72
|
+
"Add the `DB` (D1) and `MEDIA_BUCKET` (R2) bindings to each wrangler.jsonc env. `DB` is required in every mode — the dedup hash table (`pithy_media_hashes`) always lives in D1. Add the `MEDIA` KV binding when `recordStore: 'kv'`. Add the enrichment Workflow bindings your app dispatches to on finalize: `MEDIA_IMAGE_TO_TEXT`, `MEDIA_AUDIO_TRANSCRIBE`, `MEDIA_VIDEO_TRANSCRIBE`, `MEDIA_DOC_EXTRACT`.",
|
|
73
|
+
"Set the `delivery` config (public identifiers) so `mediaUrl(record, delivery)` can build consumer URLs: `imagesAccountHash` (Cloudflare Images), `streamCustomerCode` (Cloudflare Stream), and optionally `r2PublicBaseUrl` (a public R2 bucket; otherwise consumers use a presigned download URL).",
|
|
74
|
+
"Enable AI enrichment per type in `media({ ... })` in pithy.config.ts — `images.imageToText`, `video.transcribe`, `audio.transcribe`, `documents.extractText` (extraction runs for pdf/doc/docx only). Each is independently opt-in and a no-op (no cost) when off. Override any model with the type's `model` field.",
|
|
75
|
+
"The media routes require an authenticated caller. Add the auth capability (`pithy add auth`) so `c.var.auth` is populated; without it every media route is denied.",
|
|
76
|
+
"Extend a media record with your own fields — an owning `userId`, a tenant id, tags — by passing an `extend` Zod object to `media({ extend })`. The extra fields become real D1 columns (a generated migration) or a validated KV value, from that one schema. In KV mode, list them in `kvMetadata` to render them in list views without a per-value read."
|
|
77
|
+
],
|
|
78
|
+
"configOptions": [
|
|
79
|
+
{
|
|
80
|
+
"key": "recordStore",
|
|
81
|
+
"default": "d1",
|
|
82
|
+
"describe": "Where media records live: `d1` (default — transcriptions and extracted text are queryable) or `kv` (key-lookup only; no text search over derived content). Edit the per-type `images`/`video`/`audio`/`documents` blocks by hand to pick backends and toggle AI enrichment."
|
|
83
|
+
}
|
|
84
|
+
]
|
|
85
|
+
}
|
package/src/ai/enrich.ts
ADDED
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: 2026 Pithy
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
|
|
4
|
+
import { UpstreamError } from "@pithy-sh/core/src/error/pithyError";
|
|
5
|
+
import { z } from "zod";
|
|
6
|
+
import { MediaEnrichmentError } from "../error/errors";
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* The Workers AI enrichment calls, over the `env.AI` binding (bindings-first — CLAUDE.md §Cloudflare
|
|
10
|
+
* access). Each is a thin, validated wrapper the enrichment Workflows call: image-to-text (alt text and
|
|
11
|
+
* captions), speech-to-text (audio and video), and document text extraction. The model id is always a
|
|
12
|
+
* parameter, defaulted from config, so an adopter swaps a model with no code change.
|
|
13
|
+
*
|
|
14
|
+
* The binding is typed structurally as {@link MediaAi} rather than depending on the exact `Ai` shape, so
|
|
15
|
+
* a test injects a fake and the code never reaches for a global.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
/** The subset of the Workers AI binding this module uses. */
|
|
19
|
+
export interface MediaAi {
|
|
20
|
+
/** Run a model by id with an input body; the response shape varies per model, so it is `unknown`. */
|
|
21
|
+
run(model: string, input: Record<string, unknown>): Promise<unknown>;
|
|
22
|
+
/** Convert documents to markdown. Present on the AI binding; optional here so a fake can omit it. */
|
|
23
|
+
toMarkdown?: (files: MarkdownFile[], options?: Record<string, unknown>) => Promise<unknown>;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Run one call against the binding, turning a rejection into `core/upstream_failed`.
|
|
28
|
+
*
|
|
29
|
+
* **The split this enforces is between a wrong answer and no answer** (pithy-sh/pithy#348). Everything
|
|
30
|
+
* else in this module raises `media/enrichment_failed`, and every one of those is deterministic: a
|
|
31
|
+
* shape the schema does not recognize, a conversion the converter refused, a binding with no
|
|
32
|
+
* `toMarkdown`. A rejection out of the binding is the opposite — Workers AI unreachable, overloaded, or
|
|
33
|
+
* out of time — and it is the only fault here a second attempt can answer differently.
|
|
34
|
+
*
|
|
35
|
+
* The durable step can only act on that difference if it arrives as a code. A raw throw is
|
|
36
|
+
* `unclassified` to `classifyWorkflowFault`, and unclassified is terminal — so without this the one
|
|
37
|
+
* retryable fault in enrichment would have been the one that never got a retry, and an asset would
|
|
38
|
+
* silently keep no alt text because the model was busy for ten seconds.
|
|
39
|
+
*
|
|
40
|
+
* The binding's own words go in `detail`, which the HTTP codec strips: an account id or a quota
|
|
41
|
+
* message is not something a caller asking about their own file should be handed.
|
|
42
|
+
*/
|
|
43
|
+
async function callModel<T>(what: string, call: () => Promise<T>): Promise<T> {
|
|
44
|
+
try {
|
|
45
|
+
return await call();
|
|
46
|
+
} catch (error) {
|
|
47
|
+
throw new UpstreamError(
|
|
48
|
+
{ message: "The enrichment model could not be reached.", detail: `Workers AI rejected ${what}` },
|
|
49
|
+
{ cause: error },
|
|
50
|
+
);
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/** A document handed to `toMarkdown`: a name and its bytes as a Blob. */
|
|
55
|
+
export interface MarkdownFile {
|
|
56
|
+
/** The filename, including extension — the converter picks its parser from this. */
|
|
57
|
+
name: string;
|
|
58
|
+
/** The document bytes. */
|
|
59
|
+
blob: Blob;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/** The default alt-text prompt (concise, ≤ 80 tokens). Ported from the CMS. */
|
|
63
|
+
export const DEFAULT_ALT_TEXT_PROMPT = "Describe this image in a concise sentence suitable for alt text.";
|
|
64
|
+
/** The default caption prompt (longer, ≤ 512 tokens). Ported from the CMS. */
|
|
65
|
+
export const DEFAULT_CAPTION_PROMPT =
|
|
66
|
+
"Generate a concise caption for this image to be used as a description of the image.";
|
|
67
|
+
|
|
68
|
+
const ALT_TEXT_MAX_TOKENS = 80;
|
|
69
|
+
const CAPTION_MAX_TOKENS = 512;
|
|
70
|
+
|
|
71
|
+
/** Chunk size for base64-encoding audio without blowing the call stack. */
|
|
72
|
+
const BASE64_CHUNK = 0x8000;
|
|
73
|
+
|
|
74
|
+
const ImageToText = z
|
|
75
|
+
.object({ description: z.string().describe("The model's generated description of the image.") })
|
|
76
|
+
.describe("The shape of a Workers AI image-to-text response.");
|
|
77
|
+
const Transcription = z
|
|
78
|
+
.object({ text: z.string().describe("The transcribed speech.") })
|
|
79
|
+
.describe("The shape of a Workers AI speech-to-text response.");
|
|
80
|
+
const MarkdownResults = z
|
|
81
|
+
.array(
|
|
82
|
+
z
|
|
83
|
+
.object({
|
|
84
|
+
name: z.string().describe("The source document filename."),
|
|
85
|
+
format: z.string().describe("`markdown` on success, `error` on failure."),
|
|
86
|
+
data: z.string().optional().describe("The extracted markdown, when the conversion succeeded."),
|
|
87
|
+
error: z.string().optional().describe("The failure reason, when the conversion failed."),
|
|
88
|
+
})
|
|
89
|
+
.describe("One document's conversion result."),
|
|
90
|
+
)
|
|
91
|
+
.describe("The shape of a Workers AI toMarkdown response — one result per document.");
|
|
92
|
+
|
|
93
|
+
/** The alt text and caption generated for one image. */
|
|
94
|
+
export interface ImageText {
|
|
95
|
+
/** A concise sentence suitable for an `alt` attribute. */
|
|
96
|
+
altText: string;
|
|
97
|
+
/** A longer descriptive caption. */
|
|
98
|
+
caption: string;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/** Base64-encode bytes in chunks — whisper's documented `audio` input is a base64 string. */
|
|
102
|
+
function bytesToBase64(bytes: Uint8Array): string {
|
|
103
|
+
let binary = "";
|
|
104
|
+
for (let i = 0; i < bytes.length; i += BASE64_CHUNK) {
|
|
105
|
+
binary += String.fromCharCode(...bytes.subarray(i, i + BASE64_CHUNK));
|
|
106
|
+
}
|
|
107
|
+
return btoa(binary);
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/** Collapse whitespace/newlines and strip surrounding quotes from a model's text output. */
|
|
111
|
+
function cleanText(value: string): string {
|
|
112
|
+
return value
|
|
113
|
+
.replace(/[\r\n]+/g, " ")
|
|
114
|
+
.replace(/\s+/g, " ")
|
|
115
|
+
.trim()
|
|
116
|
+
.replace(/^["']+|["']+$/g, "")
|
|
117
|
+
.trim();
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/** Run the vision model once for a given prompt and return its cleaned description. */
|
|
121
|
+
async function describeImage(
|
|
122
|
+
ai: MediaAi,
|
|
123
|
+
image: Uint8Array,
|
|
124
|
+
model: string,
|
|
125
|
+
prompt: string,
|
|
126
|
+
maxTokens: number,
|
|
127
|
+
): Promise<string> {
|
|
128
|
+
const raw = await callModel(`image-to-text with model '${model}'`, () =>
|
|
129
|
+
ai.run(model, { image: [...image], prompt, max_tokens: maxTokens }),
|
|
130
|
+
);
|
|
131
|
+
const parsed = ImageToText.safeParse(raw);
|
|
132
|
+
if (!parsed.success) {
|
|
133
|
+
throw new MediaEnrichmentError({ detail: `image-to-text model '${model}' returned an unexpected shape` });
|
|
134
|
+
}
|
|
135
|
+
return cleanText(parsed.data.description);
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/**
|
|
139
|
+
* Generate alt text and a caption for an image with a vision model. Two calls (alt text is short,
|
|
140
|
+
* caption is longer). Returns both; the caller writes them to the record.
|
|
141
|
+
*/
|
|
142
|
+
export async function generateImageText(ai: MediaAi, image: Uint8Array, model: string): Promise<ImageText> {
|
|
143
|
+
const altText = await describeImage(ai, image, model, DEFAULT_ALT_TEXT_PROMPT, ALT_TEXT_MAX_TOKENS);
|
|
144
|
+
const caption = await describeImage(ai, image, model, DEFAULT_CAPTION_PROMPT, CAPTION_MAX_TOKENS);
|
|
145
|
+
return { altText, caption };
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* Transcribe audio bytes with a speech-to-text model. The bytes are base64-encoded per the model's
|
|
150
|
+
* documented `audio` string input. Returns the trimmed transcription text.
|
|
151
|
+
*/
|
|
152
|
+
export async function transcribeAudioBytes(ai: MediaAi, bytes: Uint8Array, model: string): Promise<string> {
|
|
153
|
+
const raw = await callModel(`transcription with model '${model}'`, () =>
|
|
154
|
+
ai.run(model, { audio: bytesToBase64(bytes) }),
|
|
155
|
+
);
|
|
156
|
+
const parsed = Transcription.safeParse(raw);
|
|
157
|
+
if (!parsed.success) {
|
|
158
|
+
throw new MediaEnrichmentError({ detail: `transcription model '${model}' returned an unexpected shape` });
|
|
159
|
+
}
|
|
160
|
+
return parsed.data.text.trim();
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/**
|
|
164
|
+
* Extract text (markdown) from documents via `env.AI.toMarkdown`. Joins every successful conversion,
|
|
165
|
+
* throws on any conversion error, and sanitizes the result to a safe character set (the CMS rule).
|
|
166
|
+
*/
|
|
167
|
+
export async function extractMarkdown(ai: MediaAi, files: MarkdownFile[]): Promise<string> {
|
|
168
|
+
if (!ai.toMarkdown) {
|
|
169
|
+
throw new MediaEnrichmentError({ detail: "the AI binding does not support toMarkdown" });
|
|
170
|
+
}
|
|
171
|
+
const toMarkdown = ai.toMarkdown;
|
|
172
|
+
const raw = await callModel("a document conversion", () => toMarkdown(files, { pdf: { metadata: false } }));
|
|
173
|
+
const parsed = MarkdownResults.safeParse(raw);
|
|
174
|
+
if (!parsed.success) {
|
|
175
|
+
throw new MediaEnrichmentError({ detail: "toMarkdown returned an unexpected shape" });
|
|
176
|
+
}
|
|
177
|
+
const errors = parsed.data.filter((result) => result.format === "error");
|
|
178
|
+
if (errors.length > 0) {
|
|
179
|
+
throw new MediaEnrichmentError({ detail: `toMarkdown failed: ${errors.map((e) => e.error ?? e.name).join("; ")}` });
|
|
180
|
+
}
|
|
181
|
+
const markdown = parsed.data
|
|
182
|
+
.filter((result) => result.format === "markdown")
|
|
183
|
+
.map((result) => result.data ?? "")
|
|
184
|
+
.join("\n");
|
|
185
|
+
// The CMS sanitization: keep a conservative printable set, drop control/binary noise.
|
|
186
|
+
return markdown.replace(/[^\-a-zA-Z0-9\s,.!?\n]/g, "").trim();
|
|
187
|
+
}
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: 2026 Pithy
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Pure batching and overlap-dedup for video/audio transcription. These functions carry the whole
|
|
6
|
+
* algorithm the transcription Workflow orchestrates — no Cloudflare imports, no I/O, no clock. The
|
|
7
|
+
* Workflow fetches a Cloudflare Stream HLS audio rendition, hands its segment list here to decide what
|
|
8
|
+
* to transcribe as one unit, then stitches the per-group transcripts back together here. Keeping the
|
|
9
|
+
* logic pure is deliberate: this is the unit-testable core, and the Workflow stays a thin durable shell.
|
|
10
|
+
*
|
|
11
|
+
* Ported faithfully from the leed-ai CMS `audioUtils.ts`. The grouping is whole-segment (HLS segments
|
|
12
|
+
* are the atomic unit — you can only fetch and decode a segment in full), greedy to a target length,
|
|
13
|
+
* with a deliberate backward overlap so a model that drops or garbles words at a group boundary gets a
|
|
14
|
+
* second look at that audio in the next group. The overlap is then removed from the text by
|
|
15
|
+
* `removeOverlapPrefix` so the final transcript reads clean.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
/** One audio segment from a Cloudflare Stream HLS audio rendition: its URI and playback duration. */
|
|
19
|
+
export interface AudioSegment {
|
|
20
|
+
/** The segment's URI, relative to or absolute against the rendition playlist; the fetch handle. */
|
|
21
|
+
uri: string;
|
|
22
|
+
/** The segment's playback duration in seconds, read from the HLS `#EXTINF` tag. */
|
|
23
|
+
durationSec: number;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/** A group of contiguous segments to transcribe as one unit, with the group's total duration. */
|
|
27
|
+
export interface SegmentGroup {
|
|
28
|
+
/** The contiguous segments in this group, in playlist order; transcribed together as one request. */
|
|
29
|
+
segments: AudioSegment[];
|
|
30
|
+
/** The forward-scan duration of the group in seconds — the sum accumulated before the overlap back-up. */
|
|
31
|
+
durationSec: number;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/** The length a group grows to before the greedy scan stops. Long enough for coherent context. */
|
|
35
|
+
export const TARGET_GROUP_SEC = 30;
|
|
36
|
+
|
|
37
|
+
/** The minimum audio each group re-covers from the next, so boundary words are never lost. */
|
|
38
|
+
export const OVERLAP_SEC = 3;
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Group contiguous segments into transcription units. Greedy: scan whole segments forward until the
|
|
42
|
+
* accumulated duration reaches `targetGroupSec`, emit that group, then back up over trailing segments
|
|
43
|
+
* until at least `overlapSec` of audio will be re-covered by the next group — while always leaving at
|
|
44
|
+
* least one fresh segment (`i > startIdx + 1`) so the next group makes forward progress and the loop
|
|
45
|
+
* terminates. Empty input yields no groups.
|
|
46
|
+
*
|
|
47
|
+
* `SegmentGroup.durationSec` is the forward-scan total for that group, recorded before the overlap
|
|
48
|
+
* back-up — it is not recomputed after backing up, matching the CMS behavior exactly.
|
|
49
|
+
*/
|
|
50
|
+
export function groupSegmentsForTranscription(
|
|
51
|
+
segments: AudioSegment[],
|
|
52
|
+
targetGroupSec = TARGET_GROUP_SEC,
|
|
53
|
+
overlapSec = OVERLAP_SEC,
|
|
54
|
+
): SegmentGroup[] {
|
|
55
|
+
const groups: SegmentGroup[] = [];
|
|
56
|
+
let i = 0;
|
|
57
|
+
|
|
58
|
+
while (i < segments.length) {
|
|
59
|
+
const startIdx = i;
|
|
60
|
+
let durationSec = 0;
|
|
61
|
+
|
|
62
|
+
// Forward scan: accumulate whole segments until the group reaches the target length.
|
|
63
|
+
while (i < segments.length && durationSec < targetGroupSec) {
|
|
64
|
+
const seg = segments[i];
|
|
65
|
+
if (seg === undefined) break;
|
|
66
|
+
durationSec += seg.durationSec;
|
|
67
|
+
i++;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
groups.push({ segments: segments.slice(startIdx, i), durationSec });
|
|
71
|
+
if (i >= segments.length) break;
|
|
72
|
+
|
|
73
|
+
// Back up so the next group re-covers >= overlapSec of audio, keeping >= 1 fresh segment ahead.
|
|
74
|
+
let overlapDur = 0;
|
|
75
|
+
while (i > startIdx + 1 && overlapDur < overlapSec) {
|
|
76
|
+
i--;
|
|
77
|
+
const seg = segments[i];
|
|
78
|
+
if (seg === undefined) break;
|
|
79
|
+
overlapDur += seg.durationSec;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
return groups;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* Strip from `curr` the leading words it shares with the trailing words of `prev`. Search the LONGEST
|
|
88
|
+
* overlap first: try `maxWords` trailing/leading words down to `minWords`, and on the first exact match
|
|
89
|
+
* (case-insensitive) drop that many leading words from `curr`, returning the trimmed remainder. Below
|
|
90
|
+
* `minWords` the match is too weak to trust, so on no match at any length `curr` is returned unchanged —
|
|
91
|
+
* content is never dropped on an uncertain overlap.
|
|
92
|
+
*
|
|
93
|
+
* Words split on whitespace with empty tokens ignored. Comparison lowercases both sides, but the kept
|
|
94
|
+
* words preserve `curr`'s own casing (re-joined with single spaces).
|
|
95
|
+
*/
|
|
96
|
+
export function removeOverlapPrefix(prev: string, curr: string, maxWords = 40, minWords = 3): string {
|
|
97
|
+
const prevWords = prev.split(/\s+/).filter((w) => w.length > 0);
|
|
98
|
+
const currWords = curr.split(/\s+/).filter((w) => w.length > 0);
|
|
99
|
+
const maxN = Math.min(maxWords, prevWords.length, currWords.length);
|
|
100
|
+
|
|
101
|
+
for (let n = maxN; n >= minWords; n--) {
|
|
102
|
+
const tail = prevWords.slice(prevWords.length - n);
|
|
103
|
+
const head = currWords.slice(0, n);
|
|
104
|
+
const matches = tail.every((word, idx) => word.toLowerCase() === head[idx]?.toLowerCase());
|
|
105
|
+
if (matches) return currWords.slice(n).join(" ").trim();
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
return curr;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* Stitch overlapping transcripts (one per `SegmentGroup`, in order) into one clean string. Fold left:
|
|
113
|
+
* seed with the first transcript, then for each subsequent transcript append the remainder that
|
|
114
|
+
* `removeOverlapPrefix` leaves after removing the run it shares with the text so far — joined with a
|
|
115
|
+
* single space, and only when that remainder is non-empty. The result is whitespace-normalized (runs of
|
|
116
|
+
* whitespace collapsed to single spaces, trimmed). Empty input yields an empty string.
|
|
117
|
+
*/
|
|
118
|
+
export function deduplicateOverlappingTranscripts(transcripts: string[]): string {
|
|
119
|
+
if (transcripts.length === 0) return "";
|
|
120
|
+
|
|
121
|
+
let accumulated = transcripts[0] ?? "";
|
|
122
|
+
for (let idx = 1; idx < transcripts.length; idx++) {
|
|
123
|
+
const next = transcripts[idx];
|
|
124
|
+
if (next === undefined) continue;
|
|
125
|
+
const remainder = removeOverlapPrefix(accumulated, next);
|
|
126
|
+
if (remainder.length > 0) accumulated += ` ${remainder}`;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
return accumulated.replace(/\s+/g, " ").trim();
|
|
130
|
+
}
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: 2026 Pithy
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
|
|
4
|
+
import type { BindingSpecInput } from "@pithy-sh/core/src/capability/bindings";
|
|
5
|
+
import { type Capability, defineCapability } from "@pithy-sh/core/src/capability/capability";
|
|
6
|
+
import type { DatabaseSpecMap } from "@pithy-sh/core/src/data/databases";
|
|
7
|
+
import type { KvNamespaceSpecMap } from "@pithy-sh/core/src/kv/namespaces";
|
|
8
|
+
import { workflowBindings } from "@pithy-sh/core/src/workflow/bindings";
|
|
9
|
+
import type { Migration } from "kysely/migration";
|
|
10
|
+
import type { z } from "zod";
|
|
11
|
+
import { MediaConfig, type MediaConfigInput } from "./config/config";
|
|
12
|
+
import { extendMediaAsset, extensionColumns } from "./data/extend";
|
|
13
|
+
import { mediaTables } from "./data/tables";
|
|
14
|
+
import { registerMediaRoutes } from "./http/routes";
|
|
15
|
+
import { media_0001_init } from "./migrations/0001_init";
|
|
16
|
+
import { mediaExtendMigration } from "./migrations/extend";
|
|
17
|
+
import { assertValidKvMetadata } from "./record/kvStore";
|
|
18
|
+
import { mediaSecretsRegistry } from "./secret/registry";
|
|
19
|
+
import { PACKAGE_VERSION } from "./version.generated";
|
|
20
|
+
import { mediaWorkflows } from "./workflows/specs";
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Sort order of the media migrations within the app database, relative to other capabilities (core low,
|
|
24
|
+
* app high). Unique per database; the registry composes keys like `0350_media_0001_init`.
|
|
25
|
+
*
|
|
26
|
+
* Sits after auth (300), which holds it: a media record's `ownerId` names a user, so the table that
|
|
27
|
+
* users live in should exist first. Media was itself at 300 until this was corrected — `pithy migrate`
|
|
28
|
+
* threw `duplicate migration order 300 in database "app"` for any project composing both, which is
|
|
29
|
+
* every project that stores media against an identity.
|
|
30
|
+
*/
|
|
31
|
+
export const MEDIA_MIGRATION_ORDER = 350;
|
|
32
|
+
|
|
33
|
+
/** The options `media()` accepts: the config, an optional adopter schema extension, and a mount path. */
|
|
34
|
+
export type MediaOptions = MediaConfigInput & {
|
|
35
|
+
/**
|
|
36
|
+
* Extend a media record with the adopter's own fields (an owning `userId`, a tenant id, tags), as a
|
|
37
|
+
* `z.ZodObject`. From this one schema the capability derives real D1 columns (a generated `0002_extend`
|
|
38
|
+
* migration) or a validated KV value — with no backend-specific work. Base fields are never redefined.
|
|
39
|
+
*/
|
|
40
|
+
extend?: z.ZodObject;
|
|
41
|
+
/** The path the routes mount under. Defaults to `/media`. */
|
|
42
|
+
basePath?: string;
|
|
43
|
+
};
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* The media capability, with its resolved config and effective record schema attached. The workflow
|
|
47
|
+
* slice is kept literal so a composed project types `c.var.workflows.trigger("media/image-to-text", …)`
|
|
48
|
+
* precisely — an unregistered key or a mistyped payload is a compile error, not a 500.
|
|
49
|
+
*/
|
|
50
|
+
export interface MediaCapability
|
|
51
|
+
extends Capability<DatabaseSpecMap, KvNamespaceSpecMap, "media", typeof mediaWorkflows> {
|
|
52
|
+
/** The resolved media config. */
|
|
53
|
+
mediaConfig: MediaConfig;
|
|
54
|
+
/** The effective record schema (base {@link MediaAsset} plus any adopter extension). */
|
|
55
|
+
schema: z.ZodObject;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* The media capability. It contributes the dedup `pithy_media_hashes` table to the app `DB` for **both**
|
|
60
|
+
* record stores (duplicate detection is a query workload only D1 can serve), plus the `pithy_media_assets`
|
|
61
|
+
* table when `recordStore: 'd1'` (KV mode keeps records in KV). It mounts the media routes (upload-init,
|
|
62
|
+
* finalize, get, list, delete, duplicate search — each gated by auth) and declares the enrichment Workflow
|
|
63
|
+
* bindings the finalize route dispatches to. The Workflows live in the prebuilt media worker
|
|
64
|
+
* (`workflows/worker.ts`), deployed per environment by `pithy media provision`.
|
|
65
|
+
*
|
|
66
|
+
* The `DB` binding is required in every mode (the hash table lives there). Storage credentials are read
|
|
67
|
+
* through `@pithy-sh/secrets`, so the `secrets` capability must be composed; the routes need an identity,
|
|
68
|
+
* so `@pithy-sh/auth` should be composed too — without it, `c.var.auth` is null and every route is denied.
|
|
69
|
+
*/
|
|
70
|
+
export function media(options: MediaOptions = {}): MediaCapability {
|
|
71
|
+
const { extend, basePath, ...configInput } = options;
|
|
72
|
+
const resolved = MediaConfig.parse(configInput);
|
|
73
|
+
const schema = extendMediaAsset(extend);
|
|
74
|
+
// Fail fast on a typo'd or unknown `kvMetadata` field rather than silently ignoring it.
|
|
75
|
+
assertValidKvMetadata(resolved.kvMetadata, schema);
|
|
76
|
+
const isKv = resolved.recordStore === "kv";
|
|
77
|
+
|
|
78
|
+
// One authored migration, told what to create: the hash table always (dedup is D1-only), plus the
|
|
79
|
+
// record table for the D1 record store. The extension columns are generated per adopter from their
|
|
80
|
+
// own schema, so they arrive as a second, synthesised migration rather than as part of the schema
|
|
81
|
+
// this package authors — the same shape as `@pithy-sh/auth`'s plugin tables.
|
|
82
|
+
const migrations: Record<string, Migration> = { "0001_init": media_0001_init({ withAssets: !isKv }) };
|
|
83
|
+
if (!isKv) {
|
|
84
|
+
const extendMigration = mediaExtendMigration(extensionColumns(extend));
|
|
85
|
+
if (extendMigration) migrations["0002_extend"] = extendMigration;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
const requiredBindings: BindingSpecInput[] = [
|
|
89
|
+
// The app database — the dedup hash table (both modes) and the record table (D1 mode) live here.
|
|
90
|
+
{ type: "d1", name: "DB" },
|
|
91
|
+
// The KV namespace records live in, when `recordStore: 'kv'`.
|
|
92
|
+
...(isKv ? [{ type: "kv" as const, name: "MEDIA" }] : []),
|
|
93
|
+
// The R2 bucket the routes read and delete objects through (bindings-first).
|
|
94
|
+
{ type: "r2", name: "MEDIA_BUCKET" },
|
|
95
|
+
// The enrichment Workflow bindings the finalize route dispatches to, derived from the specs rather
|
|
96
|
+
// than listed again — one declaration, so a binding rename cannot leave the two disagreeing. Each is
|
|
97
|
+
// optional: the bindings exist only once `pithy media provision` has deployed the media worker, and
|
|
98
|
+
// an unprovisioned project must still boot and serve every non-enrichment route.
|
|
99
|
+
...workflowBindings(mediaWorkflows),
|
|
100
|
+
];
|
|
101
|
+
|
|
102
|
+
const capability = defineCapability({
|
|
103
|
+
name: "media",
|
|
104
|
+
// The package version this capability ships at, stamped by `scripts/stampVersions.ts` — a Worker
|
|
105
|
+
// cannot read its own package.json. Reported per capability by the control-plane manifest.
|
|
106
|
+
version: PACKAGE_VERSION,
|
|
107
|
+
// Storage credentials are read through @pithy-sh/secrets, so secrets must be composed.
|
|
108
|
+
dependsOn: ["secrets"],
|
|
109
|
+
secretRegistry: mediaSecretsRegistry,
|
|
110
|
+
workflows: mediaWorkflows,
|
|
111
|
+
requiredBindings,
|
|
112
|
+
databases: {
|
|
113
|
+
app: {
|
|
114
|
+
binding: "DB",
|
|
115
|
+
tables: mediaTables(schema, { withAssets: !isKv }),
|
|
116
|
+
migrationOrder: MEDIA_MIGRATION_ORDER,
|
|
117
|
+
migrations,
|
|
118
|
+
},
|
|
119
|
+
},
|
|
120
|
+
routes: registerMediaRoutes({ config: resolved, schema, basePath }),
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
return Object.assign(capability, { mediaConfig: resolved, schema });
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/** Whether a capability is the media capability — carries its resolved config and effective schema. */
|
|
127
|
+
export function isMediaCapability(capability: Capability): capability is MediaCapability {
|
|
128
|
+
return capability.name === "media" && "mediaConfig" in capability;
|
|
129
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: 2026 Pithy
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
|
|
4
|
+
/// <reference types="@cloudflare/vitest-plugin/types" />
|
|
5
|
+
|
|
6
|
+
// Bindings the Workers-runtime test project provides to `*.workers.test.ts`, matching the Miniflare
|
|
7
|
+
// config in `vitest.workers.config.ts`: the app `DB` database the media tables live in, and the `MEDIA`
|
|
8
|
+
// KV namespace the `recordStore: 'kv'` path uses. `cloudflare:test` types its `env` as `Cloudflare.Env`,
|
|
9
|
+
// so test bindings are declared by augmenting that interface.
|
|
10
|
+
declare namespace Cloudflare {
|
|
11
|
+
interface Env {
|
|
12
|
+
DB: D1Database;
|
|
13
|
+
MEDIA: KVNamespace;
|
|
14
|
+
}
|
|
15
|
+
}
|