@1agh/maude 0.43.0 → 0.44.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/apps/studio/acp/bridge.ts +44 -10
- package/apps/studio/acp/env.ts +23 -0
- package/apps/studio/acp/login-state.ts +263 -0
- package/apps/studio/acp/plugin-bootstrap.ts +31 -25
- package/apps/studio/acp/probe.ts +152 -1
- package/apps/studio/annotations-layer.tsx +396 -46
- package/apps/studio/api.ts +395 -2
- package/apps/studio/bin/_audio-search.mjs +140 -0
- package/apps/studio/bin/_import-asset-pdf-worker.mjs +34 -0
- package/apps/studio/bin/_import-asset.mjs +815 -0
- package/apps/studio/bin/_import-brand.mjs +635 -0
- package/apps/studio/bin/_import-tokens-alias-resolver.mjs +95 -0
- package/apps/studio/bin/_import-tokens.mjs +1201 -0
- package/apps/studio/bin/_transcribe.mjs +419 -0
- package/apps/studio/bin/_transcribe.test.mjs +80 -0
- package/apps/studio/bin/audio-search.sh +28 -0
- package/apps/studio/bin/generate.sh +154 -0
- package/apps/studio/bin/import-asset.sh +34 -0
- package/apps/studio/bin/import-brand.sh +29 -0
- package/apps/studio/bin/import-tokens.sh +33 -0
- package/apps/studio/bin/photo-bg-remove.sh +119 -21
- package/apps/studio/bin/transcribe.sh +39 -0
- package/apps/studio/canvas-edit.ts +197 -2
- package/apps/studio/canvas-lib.tsx +115 -6
- package/apps/studio/client/app.jsx +1954 -685
- package/apps/studio/client/generate-dialog.jsx +352 -0
- package/apps/studio/client/github.js +14 -0
- package/apps/studio/client/inspector-controls.jsx +781 -0
- package/apps/studio/client/panels/BrandUploadPanel.jsx +233 -0
- package/apps/studio/client/panels/ChatPanel.jsx +15 -33
- package/apps/studio/client/panels/IntroVideoDialog.jsx +44 -0
- package/apps/studio/client/panels/OnboardingWizard.jsx +12 -0
- package/apps/studio/client/panels/ReadinessList.jsx +227 -7
- package/apps/studio/client/panels/SettingsPanel.jsx +828 -0
- package/apps/studio/client/panels/SetupChecklist.jsx +223 -0
- package/apps/studio/client/photo-knobs.jsx +167 -172
- package/apps/studio/client/styles/3-shell-maude.css +334 -28
- package/apps/studio/client/styles/4-components.css +223 -0
- package/apps/studio/client/styles/6-acp-chat.css +86 -0
- package/apps/studio/client/tour/quick-setup-tour.js +39 -0
- package/apps/studio/commands/annotation-strokes-command.ts +13 -3
- package/apps/studio/config.schema.json +70 -0
- package/apps/studio/context-menu.tsx +11 -2
- package/apps/studio/context.ts +16 -0
- package/apps/studio/design-setup-readiness.ts +115 -0
- package/apps/studio/dist/client.bundle.js +2097 -2088
- package/apps/studio/dist/styles.css +1 -1
- package/apps/studio/draw/palette.ts +34 -0
- package/apps/studio/footage/schema.test.ts +92 -0
- package/apps/studio/footage/schema.ts +179 -1
- package/apps/studio/generation/adapters/elevenlabs.ts +370 -0
- package/apps/studio/generation/adapters/gemini.ts +542 -0
- package/apps/studio/generation/adapters/groq.test.ts +80 -0
- package/apps/studio/generation/adapters/groq.ts +247 -0
- package/apps/studio/generation/audio-library.test.ts +95 -0
- package/apps/studio/generation/audio-library.ts +156 -0
- package/apps/studio/generation/captions.test.ts +78 -0
- package/apps/studio/generation/captions.ts +132 -0
- package/apps/studio/generation/download.ts +76 -0
- package/apps/studio/generation/elevenlabs.test.ts +223 -0
- package/apps/studio/generation/gemini.test.ts +350 -0
- package/apps/studio/generation/jobs.test.ts +128 -0
- package/apps/studio/generation/jobs.ts +243 -0
- package/apps/studio/generation/keys.test.ts +75 -0
- package/apps/studio/generation/keys.ts +162 -0
- package/apps/studio/generation/prefs.test.ts +95 -0
- package/apps/studio/generation/prefs.ts +82 -0
- package/apps/studio/generation/registry.ts +88 -0
- package/apps/studio/generation/types.test.ts +123 -0
- package/apps/studio/generation/types.ts +266 -0
- package/apps/studio/generation/whisper-models.test.ts +152 -0
- package/apps/studio/generation/whisper-models.ts +257 -0
- package/apps/studio/generation/whisper-spike-results.md +61 -0
- package/apps/studio/git/service.ts +3 -1
- package/apps/studio/http.ts +867 -9
- package/apps/studio/media/intro.mp4 +0 -0
- package/apps/studio/media-commit-chain.ts +119 -0
- package/apps/studio/paths.ts +10 -0
- package/apps/studio/readiness.ts +108 -20
- package/apps/studio/scaffold-design.ts +1145 -0
- package/apps/studio/server.ts +13 -1
- package/apps/studio/test/_helpers.ts +15 -2
- package/apps/studio/test/acp-bridge.test.ts +34 -1
- package/apps/studio/test/acp-env.test.ts +30 -0
- package/apps/studio/test/acp-plugin-bootstrap.test.ts +23 -57
- package/apps/studio/test/acp-session-plugins.test.ts +72 -1
- package/apps/studio/test/annotation-strokes-command.test.ts +17 -0
- package/apps/studio/test/annotations-layer.test.ts +174 -0
- package/apps/studio/test/canvas-origin-gate.test.ts +37 -0
- package/apps/studio/test/csp-canvas-shell.test.ts +11 -0
- package/apps/studio/test/csrf-write-guard.test.ts +23 -1
- package/apps/studio/test/design-setup-readiness.test.ts +155 -0
- package/apps/studio/test/element-structural-edit.test.ts +108 -0
- package/apps/studio/test/fixtures/fake-claude-auth.mjs +13 -0
- package/apps/studio/test/generate-route.test.ts +106 -0
- package/apps/studio/test/import-asset-browser.test.ts +64 -0
- package/apps/studio/test/import-asset-route.test.ts +71 -0
- package/apps/studio/test/import-asset.test.ts +399 -0
- package/apps/studio/test/import-brand-browser.test.ts +88 -0
- package/apps/studio/test/import-brand-route.test.ts +120 -0
- package/apps/studio/test/import-brand.test.ts +206 -0
- package/apps/studio/test/import-tokens.test.ts +722 -0
- package/apps/studio/test/media-commit-chain.test.ts +210 -0
- package/apps/studio/test/photo-bg-remove-validation.test.ts +71 -0
- package/apps/studio/test/readiness.test.ts +13 -11
- package/apps/studio/test/scaffold-design.test.ts +122 -0
- package/apps/studio/test/tour-overlay.test.tsx +16 -0
- package/apps/studio/tool-palette.tsx +11 -6
- package/apps/studio/ui-prefs.ts +130 -0
- package/apps/studio/use-canvas-media-drop.tsx +4 -2
- package/apps/studio/use-chrome-visibility.tsx +8 -2
- package/apps/studio/whats-new.json +96 -0
- package/apps/studio/ws.ts +5 -0
- package/cli/bin/maude.mjs +6 -6
- package/cli/commands/design.mjs +33 -0
- package/cli/lib/gitignore-block.mjs +1 -0
- package/cli/lib/pkg-root.mjs +107 -0
- package/cli/lib/pkg-root.test.mjs +79 -0
- package/cli/lib/reconstruct-toolset.test.mjs +194 -0
- package/cli/lib/update-check.mjs +8 -0
- package/package.json +12 -10
- package/plugins/design/dependencies.json +18 -0
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
// generation/registry.ts — the provider/model registry (DDR-16x). Maps a
|
|
2
|
+
// provider id → its descriptor + adapter factory, and answers capability
|
|
3
|
+
// lookups ("which providers can do image?") for routing + the Settings panel.
|
|
4
|
+
//
|
|
5
|
+
// Adding a provider = one entry here. Phase 0 ships Gemini (image); Phase 1 adds
|
|
6
|
+
// fal, Phase 2 ElevenLabs, etc. — each is a factory + descriptor, no route change.
|
|
7
|
+
//
|
|
8
|
+
// The registry holds FACTORIES, not live adapters: an adapter is instantiated
|
|
9
|
+
// per-request with the resolved key + localizer injected (AdapterContext), so a
|
|
10
|
+
// key never lives on a long-held adapter object.
|
|
11
|
+
|
|
12
|
+
import {
|
|
13
|
+
createElevenLabsAdapter,
|
|
14
|
+
ELEVENLABS_DESCRIPTOR,
|
|
15
|
+
ELEVENLABS_MODELS,
|
|
16
|
+
} from './adapters/elevenlabs.ts';
|
|
17
|
+
import { createGeminiAdapter, GEMINI_DESCRIPTOR, GEMINI_MODELS } from './adapters/gemini.ts';
|
|
18
|
+
import { createGroqAdapter, GROQ_DESCRIPTOR, GROQ_MODELS } from './adapters/groq.ts';
|
|
19
|
+
import type {
|
|
20
|
+
AdapterContext,
|
|
21
|
+
AdapterFactory,
|
|
22
|
+
Modality,
|
|
23
|
+
ModelDescriptor,
|
|
24
|
+
ProviderAdapter,
|
|
25
|
+
ProviderDescriptor,
|
|
26
|
+
} from './types.ts';
|
|
27
|
+
|
|
28
|
+
interface ProviderEntry {
|
|
29
|
+
descriptor: ProviderDescriptor;
|
|
30
|
+
factory: AdapterFactory;
|
|
31
|
+
/** Static model list (aggregators refresh via adapter.listModels at runtime). */
|
|
32
|
+
models: ModelDescriptor[];
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
const PROVIDERS: Record<string, ProviderEntry> = {
|
|
36
|
+
gemini: {
|
|
37
|
+
descriptor: GEMINI_DESCRIPTOR,
|
|
38
|
+
factory: createGeminiAdapter,
|
|
39
|
+
models: GEMINI_MODELS,
|
|
40
|
+
},
|
|
41
|
+
elevenlabs: {
|
|
42
|
+
descriptor: ELEVENLABS_DESCRIPTOR,
|
|
43
|
+
factory: createElevenLabsAdapter,
|
|
44
|
+
models: ELEVENLABS_MODELS,
|
|
45
|
+
},
|
|
46
|
+
// Task 2.6 — Groq managed Whisper STT (transcription only). A CHOSEN cloud
|
|
47
|
+
// engine, never auto-selected; the local whisper.cpp path stays the default.
|
|
48
|
+
groq: {
|
|
49
|
+
descriptor: GROQ_DESCRIPTOR,
|
|
50
|
+
factory: createGroqAdapter,
|
|
51
|
+
models: GROQ_MODELS,
|
|
52
|
+
},
|
|
53
|
+
};
|
|
54
|
+
|
|
55
|
+
/** Every registered provider descriptor (inert — safe to serialize to the UI). */
|
|
56
|
+
export function listProviders(): ProviderDescriptor[] {
|
|
57
|
+
return Object.values(PROVIDERS).map((p) => p.descriptor);
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** The descriptor for one provider, or null when unknown. */
|
|
61
|
+
export function getProviderDescriptor(id: string): ProviderDescriptor | null {
|
|
62
|
+
return PROVIDERS[id]?.descriptor ?? null;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/** The static model list for one provider. */
|
|
66
|
+
export function getProviderModels(id: string): ModelDescriptor[] {
|
|
67
|
+
return PROVIDERS[id]?.models ?? [];
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** All providers that can produce a given modality (routing + capability UI). */
|
|
71
|
+
export function providersForModality(modality: Modality): ProviderDescriptor[] {
|
|
72
|
+
return listProviders().filter((d) => d.modalities.includes(modality));
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/** Whether a provider id is registered. */
|
|
76
|
+
export function hasProvider(id: string): boolean {
|
|
77
|
+
return id in PROVIDERS;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* Instantiate the adapter for `providerId` with a per-request context. Throws on
|
|
82
|
+
* an unknown provider (the route validates first, so this is a defensive guard).
|
|
83
|
+
*/
|
|
84
|
+
export function createAdapter(providerId: string, ctx: AdapterContext): ProviderAdapter {
|
|
85
|
+
const entry = PROVIDERS[providerId];
|
|
86
|
+
if (!entry) throw new Error(`unknown provider: ${providerId}`);
|
|
87
|
+
return entry.factory(ctx);
|
|
88
|
+
}
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
// generation/types.test.ts — type-level invariants proven through a fake
|
|
2
|
+
// adapter + the request validator (the security surface behind
|
|
3
|
+
// /_api/generate-jobs). Mirrors photo/schema.test.ts's dependency-free style.
|
|
4
|
+
|
|
5
|
+
import { describe, expect, test } from 'bun:test';
|
|
6
|
+
|
|
7
|
+
import {
|
|
8
|
+
type AdapterContext,
|
|
9
|
+
type GenAsset,
|
|
10
|
+
type GenResult,
|
|
11
|
+
type Job,
|
|
12
|
+
type ProviderAdapter,
|
|
13
|
+
validateGenRequest,
|
|
14
|
+
} from './types.ts';
|
|
15
|
+
|
|
16
|
+
// A minimal sync fake — returns an already-`done` Job, proving the contract is
|
|
17
|
+
// implementable without branching on sync/async.
|
|
18
|
+
function fakeAdapter(_ctx: AdapterContext): ProviderAdapter {
|
|
19
|
+
return {
|
|
20
|
+
descriptor: {
|
|
21
|
+
id: 'fake',
|
|
22
|
+
label: 'Fake',
|
|
23
|
+
kind: 'cloud',
|
|
24
|
+
auth: 'api-key',
|
|
25
|
+
keychainService: 'com.maude.app.fake',
|
|
26
|
+
modalities: ['image'],
|
|
27
|
+
},
|
|
28
|
+
async submit() {
|
|
29
|
+
const result: GenResult = {
|
|
30
|
+
assets: [{ kind: 'image', mime: 'image/png', bytes: new Uint8Array([1, 2, 3]) }],
|
|
31
|
+
usage: { ms: 1 },
|
|
32
|
+
};
|
|
33
|
+
const job: Job = {
|
|
34
|
+
id: 'job_1',
|
|
35
|
+
status: () => 'done',
|
|
36
|
+
async *events() {
|
|
37
|
+
yield { status: 'done' as const };
|
|
38
|
+
},
|
|
39
|
+
result: async () => result,
|
|
40
|
+
cancel: () => {},
|
|
41
|
+
};
|
|
42
|
+
return job;
|
|
43
|
+
},
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
describe('generation contract', () => {
|
|
48
|
+
test('a sync adapter returns an already-done Job whose result localizes', async () => {
|
|
49
|
+
const localized: GenAsset[] = [];
|
|
50
|
+
const ctx: AdapterContext = {
|
|
51
|
+
apiKey: 'k',
|
|
52
|
+
localize: async (a) => {
|
|
53
|
+
localized.push(a);
|
|
54
|
+
return 'assets/deadbeef.png';
|
|
55
|
+
},
|
|
56
|
+
};
|
|
57
|
+
const adapter = fakeAdapter(ctx);
|
|
58
|
+
const job = await adapter.submit({ modality: 'image', provider: 'fake', prompt: 'a circle' });
|
|
59
|
+
expect(job.status()).toBe('done');
|
|
60
|
+
const res = await job.result();
|
|
61
|
+
expect(res.assets).toHaveLength(1);
|
|
62
|
+
const rel = await ctx.localize(res.assets[0]);
|
|
63
|
+
expect(rel).toBe('assets/deadbeef.png');
|
|
64
|
+
expect(localized).toHaveLength(1);
|
|
65
|
+
});
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
describe('validateGenRequest', () => {
|
|
69
|
+
test('accepts a well-formed image request', () => {
|
|
70
|
+
const r = validateGenRequest({ modality: 'image', provider: 'gemini', prompt: 'a red circle' });
|
|
71
|
+
expect(r.ok).toBe(true);
|
|
72
|
+
expect(r.errors).toEqual([]);
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
test('accepts a source-only edit request (no prompt)', () => {
|
|
76
|
+
const r = validateGenRequest({
|
|
77
|
+
modality: 'transcription',
|
|
78
|
+
provider: 'whisper',
|
|
79
|
+
sourceAsset: 'assets/abcd1234.mp4',
|
|
80
|
+
});
|
|
81
|
+
expect(r.ok).toBe(true);
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
test('rejects a non-object', () => {
|
|
85
|
+
expect(validateGenRequest(null).ok).toBe(false);
|
|
86
|
+
expect(validateGenRequest('x').ok).toBe(false);
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
test('rejects unknown modality + malformed provider', () => {
|
|
90
|
+
const r = validateGenRequest({ modality: 'hologram', provider: '../etc' });
|
|
91
|
+
expect(r.ok).toBe(false);
|
|
92
|
+
expect(r.errors.some((e) => e.startsWith('modality'))).toBe(true);
|
|
93
|
+
expect(r.errors.some((e) => e.startsWith('provider'))).toBe(true);
|
|
94
|
+
});
|
|
95
|
+
|
|
96
|
+
test('rejects a traversing sourceAsset', () => {
|
|
97
|
+
const r = validateGenRequest({
|
|
98
|
+
modality: 'image',
|
|
99
|
+
provider: 'gemini',
|
|
100
|
+
sourceAsset: '../../etc/passwd',
|
|
101
|
+
});
|
|
102
|
+
expect(r.ok).toBe(false);
|
|
103
|
+
expect(r.errors.some((e) => e.startsWith('sourceAsset'))).toBe(true);
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
test('rejects a bad aspect ratio + an over-long prompt', () => {
|
|
107
|
+
const r = validateGenRequest({
|
|
108
|
+
modality: 'image',
|
|
109
|
+
provider: 'gemini',
|
|
110
|
+
prompt: 'x'.repeat(9000),
|
|
111
|
+
aspectRatio: 'wide',
|
|
112
|
+
});
|
|
113
|
+
expect(r.ok).toBe(false);
|
|
114
|
+
expect(r.errors.some((e) => e.startsWith('prompt'))).toBe(true);
|
|
115
|
+
expect(r.errors.some((e) => e.startsWith('aspectRatio'))).toBe(true);
|
|
116
|
+
});
|
|
117
|
+
|
|
118
|
+
test('rejects a request with neither prompt nor sourceAsset', () => {
|
|
119
|
+
const r = validateGenRequest({ modality: 'image', provider: 'gemini' });
|
|
120
|
+
expect(r.ok).toBe(false);
|
|
121
|
+
expect(r.errors.some((e) => e.includes('prompt or a sourceAsset'))).toBe(true);
|
|
122
|
+
});
|
|
123
|
+
});
|
|
@@ -0,0 +1,266 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file generation/types.ts — the provider-adapter contract (DDR-16x).
|
|
3
|
+
* @scope apps/studio/generation/types.ts
|
|
4
|
+
* @purpose The single normalized contract every media-generation provider —
|
|
5
|
+
* cloud aggregator (fal), direct-BYOK provider (Gemini/ElevenLabs),
|
|
6
|
+
* or local runtime (Ollama/ComfyUI) — is expressed through:
|
|
7
|
+
* `submit(req) → Job`, and the produced bytes localize into the
|
|
8
|
+
* content-addressed `assets/<sha8>` store (DDR-088).
|
|
9
|
+
*
|
|
10
|
+
* @invariant DEPENDENCY-FREE. Imported by BOTH the server (registry, adapters,
|
|
11
|
+
* the /_api/generate-jobs route) and — for the descriptor/model
|
|
12
|
+
* shapes only — the client Settings panel. It MUST NOT import any
|
|
13
|
+
* provider SDK, `node:*`, or browser lib. Pure TS types + plain
|
|
14
|
+
* hand-rolled validators (no Ajv), mirroring `photo/schema.ts` and
|
|
15
|
+
* `footage/schema.ts`.
|
|
16
|
+
*
|
|
17
|
+
* Key custody lives in `keys.ts`, NOT here — a descriptor only
|
|
18
|
+
* DECLARES how it authenticates (`auth` + `keychainService`); the
|
|
19
|
+
* host injects the resolved credential at call time so keys stay in
|
|
20
|
+
* one place and never cross into the untrusted canvas realm (DDR-054).
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
/** Schema version of the generation contract — bumped on incompatible change. */
|
|
24
|
+
export const GENERATION_CONTRACT_VERSION = 1 as const;
|
|
25
|
+
|
|
26
|
+
/** What a provider produces. */
|
|
27
|
+
export type Modality = 'image' | 'video' | 'audio' | 'transcription';
|
|
28
|
+
export const MODALITIES: readonly Modality[] = ['image', 'video', 'audio', 'transcription'];
|
|
29
|
+
|
|
30
|
+
/** Where a provider runs — a `cloud` call needs a key + egress; `local` is a
|
|
31
|
+
* loopback runtime on the user's own hardware (post-v1). */
|
|
32
|
+
export type ProviderKind = 'cloud' | 'local';
|
|
33
|
+
|
|
34
|
+
/** How a provider authenticates. `none` = local runtime (no credential). */
|
|
35
|
+
export type AuthKind = 'api-key' | 'none';
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* A provider — a vendor/runtime that exposes one or more models across one or
|
|
39
|
+
* more modalities. The descriptor is inert data (safe to serialize to the
|
|
40
|
+
* client Settings panel); it carries NO secret.
|
|
41
|
+
*/
|
|
42
|
+
export interface ProviderDescriptor {
|
|
43
|
+
/** Stable slug — the key custody + config + route all key off this. */
|
|
44
|
+
id: string;
|
|
45
|
+
/** Human label for the Settings UI. */
|
|
46
|
+
label: string;
|
|
47
|
+
kind: ProviderKind;
|
|
48
|
+
auth: AuthKind;
|
|
49
|
+
/**
|
|
50
|
+
* OS-keychain service name for the native tier (`com.maude.app.<id>`), and
|
|
51
|
+
* the lookup key in the `~/.config/maude/keys.json` browser fallback. Absent
|
|
52
|
+
* for `auth: 'none'` local runtimes.
|
|
53
|
+
*/
|
|
54
|
+
keychainService?: string;
|
|
55
|
+
/** Modalities this provider can produce (drives capability lookup + routing). */
|
|
56
|
+
modalities: readonly Modality[];
|
|
57
|
+
/**
|
|
58
|
+
* Where the user gets a key + the pricing/licensing note the Settings UI must
|
|
59
|
+
* surface (music commercial-rights tier, voice-cloning consent, etc.). Inert
|
|
60
|
+
* copy, never a secret.
|
|
61
|
+
*/
|
|
62
|
+
keyUrl?: string;
|
|
63
|
+
notes?: string;
|
|
64
|
+
/** Default loopback host:port for a `local` runtime (probe target). */
|
|
65
|
+
localEndpoint?: string;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/** One concrete model a provider offers. */
|
|
69
|
+
export interface ModelDescriptor {
|
|
70
|
+
/** Provider-scoped id passed to `submit` (e.g. `gemini-2.5-flash-image`). */
|
|
71
|
+
id: string;
|
|
72
|
+
label: string;
|
|
73
|
+
modality: Modality;
|
|
74
|
+
/** Non-secret capability hints for the generate dialog + routing. */
|
|
75
|
+
aspectRatios?: readonly string[];
|
|
76
|
+
/** Rough per-unit cost note for the UI cost badge (`~$0.04/image`). */
|
|
77
|
+
costNote?: string;
|
|
78
|
+
/** True when the model round-trips synchronously (Job returns already `done`). */
|
|
79
|
+
sync?: boolean;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/** The user's generation request, normalized across every modality. */
|
|
83
|
+
export interface GenRequest {
|
|
84
|
+
modality: Modality;
|
|
85
|
+
/** Provider id (`gemini`, `elevenlabs`, `fal`, …). */
|
|
86
|
+
provider: string;
|
|
87
|
+
/** Provider-scoped model id. Optional — adapter picks its default. */
|
|
88
|
+
model?: string;
|
|
89
|
+
/** The text prompt (image/video/audio) or the source asset (transcription). */
|
|
90
|
+
prompt?: string;
|
|
91
|
+
/** `assets/<sha8>.<ext>` source for edit/i2v/transcription flows. */
|
|
92
|
+
sourceAsset?: string;
|
|
93
|
+
/** `1:1` / `16:9` / … — adapter maps to the provider's own vocabulary. */
|
|
94
|
+
aspectRatio?: string;
|
|
95
|
+
/** Free-form provider-specific knobs (duration, voice_id, seed, …). Opaque
|
|
96
|
+
* to the spine; each adapter validates its own subset. */
|
|
97
|
+
params?: Record<string, unknown>;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
export type JobStatus = 'queued' | 'running' | 'done' | 'failed';
|
|
101
|
+
|
|
102
|
+
/** A progress/log tick a long-running (async) job may emit. */
|
|
103
|
+
export interface JobEvent {
|
|
104
|
+
status: JobStatus;
|
|
105
|
+
/** 0..1 fractional progress when the provider reports it. */
|
|
106
|
+
progress?: number;
|
|
107
|
+
message?: string;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/** One produced artifact — a URL (localize immediately, may expire), inline
|
|
111
|
+
* base64/bytes, or a caption text blob. Exactly one payload field is set. */
|
|
112
|
+
export interface GenAsset {
|
|
113
|
+
kind: Modality;
|
|
114
|
+
mime: string;
|
|
115
|
+
/** Expiring provider URL — MUST be downloaded through the hardened egress path
|
|
116
|
+
* (`download.ts`) before it's referenced anywhere. */
|
|
117
|
+
url?: string;
|
|
118
|
+
/** Inline bytes (sync base64 providers decode into this). */
|
|
119
|
+
bytes?: Uint8Array;
|
|
120
|
+
/** Text payload (SRT/VTT captions from a transcription provider). */
|
|
121
|
+
text?: string;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/** The normalized result once a Job is `done`. */
|
|
125
|
+
export interface GenResult {
|
|
126
|
+
assets: GenAsset[];
|
|
127
|
+
usage?: { cost?: number; ms?: number };
|
|
128
|
+
/** The provider's raw response, kept for audit/debug (never sent to canvas). */
|
|
129
|
+
raw?: unknown;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* A generation handle. Sync providers return an already-`done` Job (so callers
|
|
134
|
+
* never branch on sync/async); async providers drive `queued → running → done`.
|
|
135
|
+
*/
|
|
136
|
+
export interface Job {
|
|
137
|
+
id: string;
|
|
138
|
+
status(): JobStatus;
|
|
139
|
+
/** Live progress stream (async providers). Sync providers yield one `done`. */
|
|
140
|
+
events(): AsyncIterable<JobEvent>;
|
|
141
|
+
result(): Promise<GenResult>;
|
|
142
|
+
cancel(): void;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/**
|
|
146
|
+
* One re-usable past generation from a provider's history (Task 2.5). The user's
|
|
147
|
+
* OWN prior audio, re-downloadable for NO credit (already paid) — ranked against
|
|
148
|
+
* a query alongside the project's local generated audio so reuse beats paying
|
|
149
|
+
* for a near-identical new generation.
|
|
150
|
+
*/
|
|
151
|
+
export interface HistoryAudioItem {
|
|
152
|
+
/** Provider-scoped id used to re-fetch the bytes (`fetchHistoryAudio`). */
|
|
153
|
+
id: string;
|
|
154
|
+
/** The stored source text / prompt this item was generated from (match key). */
|
|
155
|
+
text: string;
|
|
156
|
+
/** ISO timestamp when it was produced (recency tie-break). */
|
|
157
|
+
at?: string;
|
|
158
|
+
/** e.g. a voice_id for TTS — surfaced for provenance, never trusted blindly. */
|
|
159
|
+
voiceId?: string;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/**
|
|
163
|
+
* The one interface every adapter implements. `submit` returns immediately —
|
|
164
|
+
* even sync providers hand back a resolved Job so the host queue is uniform.
|
|
165
|
+
*/
|
|
166
|
+
export interface ProviderAdapter {
|
|
167
|
+
descriptor: ProviderDescriptor;
|
|
168
|
+
/** Refresh the model catalogue (aggregators). Static providers may omit it. */
|
|
169
|
+
listModels?(): Promise<ModelDescriptor[]>;
|
|
170
|
+
submit(req: GenRequest): Promise<Job>;
|
|
171
|
+
/**
|
|
172
|
+
* Task 2.5 — list the user's re-usable audio history (ElevenLabs). Optional;
|
|
173
|
+
* only providers with a free re-download history implement it. Used by the
|
|
174
|
+
* audio-search route to offer reuse before a paid generation.
|
|
175
|
+
*/
|
|
176
|
+
listHistory?(): Promise<HistoryAudioItem[]>;
|
|
177
|
+
/**
|
|
178
|
+
* Task 2.5 — re-fetch a history item's bytes (NO credit — already paid) as an
|
|
179
|
+
* audio GenAsset the host localizes into `assets/<sha8>`. Optional, paired
|
|
180
|
+
* with `listHistory`.
|
|
181
|
+
*/
|
|
182
|
+
fetchHistoryAudio?(id: string): Promise<GenAsset>;
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
/**
|
|
186
|
+
* Context the host injects into `submit` at call time — the resolved credential
|
|
187
|
+
* (never stored on the adapter) + a localizer that lands result bytes/URLs into
|
|
188
|
+
* `assets/<sha8>`. Passed alongside the request so key custody stays external.
|
|
189
|
+
*/
|
|
190
|
+
export interface AdapterContext {
|
|
191
|
+
/** The resolved API key, or null for a `local`/`none`-auth provider. */
|
|
192
|
+
apiKey: string | null;
|
|
193
|
+
/** Localize a produced asset into `assets/<sha8>.<ext>`; returns the rel path. */
|
|
194
|
+
localize(asset: GenAsset): Promise<string>;
|
|
195
|
+
/**
|
|
196
|
+
* Read a content-addressed `assets/<sha8>.<ext>` source's bytes + sniffed mime
|
|
197
|
+
* for edit / image-to-video flows (a `GenRequest.sourceAsset`). Host-provided
|
|
198
|
+
* so the adapter never touches the filesystem; returns null for an unknown or
|
|
199
|
+
* contained-out path. Absent when the host wires no source access (Phase 0).
|
|
200
|
+
*/
|
|
201
|
+
readSourceAsset?(rel: string): Promise<{ bytes: Uint8Array; mime: string } | null>;
|
|
202
|
+
/** AbortSignal so a cancelled job tears down its provider request. */
|
|
203
|
+
signal?: AbortSignal;
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
/** An adapter factory — the registry stores these, one per provider. */
|
|
207
|
+
export type AdapterFactory = (ctx: AdapterContext) => ProviderAdapter;
|
|
208
|
+
|
|
209
|
+
// ── Structural validation (dependency-free — no Ajv) ─────────────────────────
|
|
210
|
+
// The /_api/generate-jobs route validates the untrusted request body with this
|
|
211
|
+
// before it reaches an adapter (security surface — a crafted field must never
|
|
212
|
+
// reach a provider POST or the filesystem). Mirrors photo/schema.ts's style.
|
|
213
|
+
|
|
214
|
+
export interface ValidationResult {
|
|
215
|
+
ok: boolean;
|
|
216
|
+
errors: string[];
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
const isPlainObject = (v: unknown): v is Record<string, unknown> =>
|
|
220
|
+
typeof v === 'object' && v !== null && !Array.isArray(v);
|
|
221
|
+
|
|
222
|
+
// Same content-addressed relative-path allowlist as photo/schema.ts — never a
|
|
223
|
+
// data:/blob:/absolute/traversing path.
|
|
224
|
+
const ASSET_REL_RE = /^assets\/[0-9a-f]{8,}[A-Za-z0-9._-]*$/;
|
|
225
|
+
// Conservative provider/model id shape — slug-ish, no path separators.
|
|
226
|
+
const ID_RE = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/;
|
|
227
|
+
const ASPECT_RE = /^\d{1,2}:\d{1,2}$/;
|
|
228
|
+
const MAX_PROMPT_LEN = 8000;
|
|
229
|
+
|
|
230
|
+
export function isModality(v: unknown): v is Modality {
|
|
231
|
+
return typeof v === 'string' && (MODALITIES as readonly string[]).includes(v);
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
/** Validate an untrusted `GenRequest` body. Returns collected errors. */
|
|
235
|
+
export function validateGenRequest(input: unknown): ValidationResult {
|
|
236
|
+
const errors: string[] = [];
|
|
237
|
+
if (!isPlainObject(input)) return { ok: false, errors: ['request must be an object'] };
|
|
238
|
+
|
|
239
|
+
if (!isModality(input.modality)) errors.push(`modality: must be one of ${MODALITIES.join('|')}`);
|
|
240
|
+
if (typeof input.provider !== 'string' || !ID_RE.test(input.provider))
|
|
241
|
+
errors.push('provider: missing or malformed id');
|
|
242
|
+
if (input.model != null && (typeof input.model !== 'string' || !ID_RE.test(input.model)))
|
|
243
|
+
errors.push('model: malformed id');
|
|
244
|
+
|
|
245
|
+
if (input.prompt != null) {
|
|
246
|
+
if (typeof input.prompt !== 'string') errors.push('prompt: must be a string');
|
|
247
|
+
else if (input.prompt.length > MAX_PROMPT_LEN)
|
|
248
|
+
errors.push(`prompt: exceeds ${MAX_PROMPT_LEN} chars`);
|
|
249
|
+
}
|
|
250
|
+
if (input.sourceAsset != null) {
|
|
251
|
+
if (typeof input.sourceAsset !== 'string' || !ASSET_REL_RE.test(input.sourceAsset))
|
|
252
|
+
errors.push('sourceAsset: must be a content-addressed assets/<sha8>.<ext> path');
|
|
253
|
+
}
|
|
254
|
+
if (input.aspectRatio != null) {
|
|
255
|
+
if (typeof input.aspectRatio !== 'string' || !ASPECT_RE.test(input.aspectRatio))
|
|
256
|
+
errors.push('aspectRatio: must look like W:H');
|
|
257
|
+
}
|
|
258
|
+
if (input.params != null && !isPlainObject(input.params))
|
|
259
|
+
errors.push('params: must be an object');
|
|
260
|
+
|
|
261
|
+
// A request needs SOMETHING to act on — a prompt or a source asset.
|
|
262
|
+
if (input.prompt == null && input.sourceAsset == null)
|
|
263
|
+
errors.push('request needs a prompt or a sourceAsset');
|
|
264
|
+
|
|
265
|
+
return { ok: errors.length === 0, errors };
|
|
266
|
+
}
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
// whisper-models.test.ts — managed local whisper.cpp model registry + resolution
|
|
2
|
+
// + download (Task 2.7, approach A). Uses a temp XDG_CACHE_HOME so the real
|
|
3
|
+
// user cache is never touched, and a stubbed fetch for the download path.
|
|
4
|
+
|
|
5
|
+
import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
|
|
6
|
+
import { existsSync, mkdtempSync, writeFileSync } from 'node:fs';
|
|
7
|
+
import { tmpdir } from 'node:os';
|
|
8
|
+
import { join } from 'node:path';
|
|
9
|
+
|
|
10
|
+
import {
|
|
11
|
+
downloadWhisperModel,
|
|
12
|
+
getWhisperModel,
|
|
13
|
+
listWhisperModels,
|
|
14
|
+
removeWhisperModel,
|
|
15
|
+
resolveWhisperModel,
|
|
16
|
+
WHISPER_MODELS,
|
|
17
|
+
whisperModelsDir,
|
|
18
|
+
whisperModelUrl,
|
|
19
|
+
} from './whisper-models.ts';
|
|
20
|
+
|
|
21
|
+
const realFetch = globalThis.fetch;
|
|
22
|
+
let cacheRoot: string;
|
|
23
|
+
const prevXdg = process.env.XDG_CACHE_HOME;
|
|
24
|
+
|
|
25
|
+
beforeEach(() => {
|
|
26
|
+
cacheRoot = mkdtempSync(join(tmpdir(), 'maude-wm-'));
|
|
27
|
+
process.env.XDG_CACHE_HOME = cacheRoot;
|
|
28
|
+
});
|
|
29
|
+
afterEach(() => {
|
|
30
|
+
globalThis.fetch = realFetch;
|
|
31
|
+
if (prevXdg === undefined) delete process.env.XDG_CACHE_HOME;
|
|
32
|
+
else process.env.XDG_CACHE_HOME = prevXdg;
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
describe('registry', () => {
|
|
36
|
+
test('carries base (multilingual) + base.en (English-only)', () => {
|
|
37
|
+
expect(getWhisperModel('base')?.multilingual).toBe(true);
|
|
38
|
+
expect(getWhisperModel('base.en')?.multilingual).toBe(false);
|
|
39
|
+
expect(getWhisperModel('nope')).toBeNull();
|
|
40
|
+
});
|
|
41
|
+
test('whisperModelUrl derives the frozen ggerganov/whisper.cpp URL (SSRF-safe host)', () => {
|
|
42
|
+
const url = whisperModelUrl(getWhisperModel('base'));
|
|
43
|
+
expect(url).toBe('https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.bin');
|
|
44
|
+
});
|
|
45
|
+
test('whisperModelsDir honors XDG_CACHE_HOME', () => {
|
|
46
|
+
expect(whisperModelsDir()).toBe(join(cacheRoot, 'maude', 'whisper-models'));
|
|
47
|
+
});
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
describe('resolveWhisperModel', () => {
|
|
51
|
+
function fakeDownloaded(file: string) {
|
|
52
|
+
const dir = whisperModelsDir();
|
|
53
|
+
require('node:fs').mkdirSync(dir, { recursive: true });
|
|
54
|
+
writeFileSync(join(dir, file), 'x');
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
test('returns null when nothing is downloaded', () => {
|
|
58
|
+
expect(resolveWhisperModel()).toBeNull();
|
|
59
|
+
});
|
|
60
|
+
test('prefers the configured id when present', () => {
|
|
61
|
+
fakeDownloaded('ggml-small.bin');
|
|
62
|
+
fakeDownloaded('ggml-base.bin');
|
|
63
|
+
expect(resolveWhisperModel('small')).toBe(join(whisperModelsDir(), 'ggml-small.bin'));
|
|
64
|
+
});
|
|
65
|
+
test('prefers a MULTILINGUAL model over an English-only one (Czech-footage safety)', () => {
|
|
66
|
+
fakeDownloaded('ggml-base.en.bin'); // English-only
|
|
67
|
+
fakeDownloaded('ggml-base.bin'); // multilingual
|
|
68
|
+
expect(resolveWhisperModel()).toBe(join(whisperModelsDir(), 'ggml-base.bin'));
|
|
69
|
+
});
|
|
70
|
+
test('falls back to any model if only English-only is present', () => {
|
|
71
|
+
fakeDownloaded('ggml-base.en.bin');
|
|
72
|
+
expect(resolveWhisperModel()).toBe(join(whisperModelsDir(), 'ggml-base.en.bin'));
|
|
73
|
+
});
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
describe('listWhisperModels', () => {
|
|
77
|
+
test('flags a downloaded model', () => {
|
|
78
|
+
const dir = whisperModelsDir();
|
|
79
|
+
require('node:fs').mkdirSync(dir, { recursive: true });
|
|
80
|
+
writeFileSync(join(dir, 'ggml-tiny.bin'), 'abc');
|
|
81
|
+
const tiny = listWhisperModels().find((m) => m.id === 'tiny');
|
|
82
|
+
expect(tiny?.downloaded).toBe(true);
|
|
83
|
+
expect(tiny?.bytes).toBe(3);
|
|
84
|
+
expect(listWhisperModels().find((m) => m.id === 'base')?.downloaded).toBe(false);
|
|
85
|
+
});
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
describe('downloadWhisperModel', () => {
|
|
89
|
+
test('streams to the cache, reports progress, atomically lands the file', async () => {
|
|
90
|
+
const bytes = new Uint8Array(1024);
|
|
91
|
+
globalThis.fetch = (async () =>
|
|
92
|
+
new Response(bytes, {
|
|
93
|
+
status: 200,
|
|
94
|
+
headers: { 'content-length': String(bytes.byteLength) },
|
|
95
|
+
})) as typeof fetch;
|
|
96
|
+
const seen: number[] = [];
|
|
97
|
+
const path = await downloadWhisperModel('tiny', (r) => seen.push(r));
|
|
98
|
+
expect(path).toBe(join(whisperModelsDir(), 'ggml-tiny.bin'));
|
|
99
|
+
expect(existsSync(path)).toBe(true);
|
|
100
|
+
expect(existsSync(`${path}.part`)).toBe(false); // temp renamed away
|
|
101
|
+
expect(seen.at(-1)).toBe(1024);
|
|
102
|
+
expect(await removeWhisperModel('tiny')).toBe(true);
|
|
103
|
+
expect(existsSync(path)).toBe(false);
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
test('rejects an over-cap body (declared content-length) without writing', async () => {
|
|
107
|
+
globalThis.fetch = (async () =>
|
|
108
|
+
new Response(new Uint8Array(1), {
|
|
109
|
+
status: 200,
|
|
110
|
+
headers: { 'content-length': String(9999 * 1024 * 1024) }, // way over tiny's cap
|
|
111
|
+
})) as typeof fetch;
|
|
112
|
+
await expect(downloadWhisperModel('tiny', () => {})).rejects.toThrow(/larger than expected/);
|
|
113
|
+
expect(existsSync(join(whisperModelsDir(), 'ggml-tiny.bin'))).toBe(false);
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
test('unknown model id throws', async () => {
|
|
117
|
+
await expect(downloadWhisperModel('bogus', () => {})).rejects.toThrow(/unknown whisper model/);
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
test('rejects a redirect that lands off huggingface.co (Finding 2)', async () => {
|
|
121
|
+
// Simulate the HF blob redirect resolving to an attacker/on-path host.
|
|
122
|
+
globalThis.fetch = (async () => ({
|
|
123
|
+
url: 'https://evil.example/ggml-tiny.bin',
|
|
124
|
+
ok: true,
|
|
125
|
+
headers: new Headers(),
|
|
126
|
+
body: new Response(new Uint8Array(1)).body,
|
|
127
|
+
})) as unknown as typeof fetch;
|
|
128
|
+
await expect(downloadWhisperModel('tiny', () => {})).rejects.toThrow(/off huggingface\.co/);
|
|
129
|
+
expect(existsSync(join(whisperModelsDir(), 'ggml-tiny.bin'))).toBe(false);
|
|
130
|
+
});
|
|
131
|
+
|
|
132
|
+
test('accepts a redirect that stays on a huggingface.co CDN host', async () => {
|
|
133
|
+
const bytes = new Uint8Array(512);
|
|
134
|
+
globalThis.fetch = (async () => ({
|
|
135
|
+
url: 'https://cdn-lfs.huggingface.co/repos/x/ggml-tiny.bin',
|
|
136
|
+
ok: true,
|
|
137
|
+
headers: new Headers({ 'content-length': String(bytes.byteLength) }),
|
|
138
|
+
body: new Response(bytes).body,
|
|
139
|
+
})) as unknown as typeof fetch;
|
|
140
|
+
const path = await downloadWhisperModel('tiny', () => {});
|
|
141
|
+
expect(existsSync(path)).toBe(true);
|
|
142
|
+
});
|
|
143
|
+
});
|
|
144
|
+
|
|
145
|
+
describe('registry integrity', () => {
|
|
146
|
+
test('every model has a positive size and a ggml- file', () => {
|
|
147
|
+
for (const m of WHISPER_MODELS) {
|
|
148
|
+
expect(m.sizeMB).toBeGreaterThan(0);
|
|
149
|
+
expect(m.file.startsWith('ggml-')).toBe(true);
|
|
150
|
+
}
|
|
151
|
+
});
|
|
152
|
+
});
|