dsh-hyperframes 0.10.1 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +13 -0
- package/README.en.md +4 -3
- package/README.md +4 -3
- package/docs/WORKBENCH.md +6 -0
- package/lib/client.js +40 -2
- package/lib/media-templates.d.ts +1 -1
- package/lib/media-templates.js +1 -1
- package/package.json +2 -2
- package/plugin.json +1 -1
- package/skills/NOTICE.md +1 -1
- package/skills/UPSTREAM.json +93 -8
- package/skills/embedded-captions/SKILL.md +1 -1
- package/skills/faceless-explainer/SKILL.md +1 -1
- package/skills/general-video/SKILL.md +1 -1
- package/skills/hyperframes-animation/SKILL.md +1 -1
- package/skills/hyperframes-audio/SKILL.md +1 -1
- package/skills/hyperframes-core/SKILL.md +1 -1
- package/skills/hyperframes-creative/SKILL.md +1 -1
- package/skills/hyperframes-keyframes/SKILL.md +1 -1
- package/skills/hyperframes-studio/SKILL.md +1 -1
- package/skills/media-use/scripts/lib/adopt.mjs +128 -0
- package/skills/media-use/scripts/lib/adopt.test.mjs +95 -0
- package/skills/media-use/scripts/lib/bgm-provider.mjs +20 -0
- package/skills/media-use/scripts/lib/brand-provider.mjs +59 -0
- package/skills/media-use/scripts/lib/bundled-sfx-provider.mjs +119 -0
- package/skills/media-use/scripts/lib/bundled-sfx-provider.test.mjs +139 -0
- package/skills/media-use/scripts/lib/cache.mjs +162 -0
- package/skills/media-use/scripts/lib/cache.test.mjs +110 -0
- package/skills/media-use/scripts/lib/candidates.mjs +91 -0
- package/skills/media-use/scripts/lib/candidates.test.mjs +155 -0
- package/skills/media-use/scripts/lib/codex-provider.mjs +161 -0
- package/skills/media-use/scripts/lib/codex-provider.test.mjs +34 -0
- package/skills/media-use/scripts/lib/config-lock.d.mts +3 -0
- package/skills/media-use/scripts/lib/config-lock.test.mjs +79 -0
- package/skills/media-use/scripts/lib/coverage.test.mjs +136 -0
- package/skills/media-use/scripts/lib/cube-build.mjs +181 -0
- package/skills/media-use/scripts/lib/cube-build.test.mjs +80 -0
- package/skills/media-use/scripts/lib/cube-validate.mjs +197 -0
- package/skills/media-use/scripts/lib/cube-validate.test.mjs +125 -0
- package/skills/media-use/scripts/lib/cutlist.test.mjs +174 -0
- package/skills/media-use/scripts/lib/duck.test.mjs +319 -0
- package/skills/media-use/scripts/lib/error-diffusion.test.mjs +102 -0
- package/skills/media-use/scripts/lib/ff-binaries.mjs +37 -0
- package/skills/media-use/scripts/lib/freeze.mjs +72 -0
- package/skills/media-use/scripts/lib/freeze.test.mjs +138 -0
- package/skills/media-use/scripts/lib/grade-analyzer.mjs +277 -0
- package/skills/media-use/scripts/lib/grade-analyzer.test.mjs +285 -0
- package/skills/media-use/scripts/lib/heygen-cli.mjs +208 -0
- package/skills/media-use/scripts/lib/heygen-cli.test.mjs +315 -0
- package/skills/media-use/scripts/lib/heygen-search.mjs +45 -0
- package/skills/media-use/scripts/lib/heygen-search.test.mjs +49 -0
- package/skills/media-use/scripts/lib/heygen-video-provider.mjs +138 -0
- package/skills/media-use/scripts/lib/heygen-video-provider.test.mjs +435 -0
- package/skills/media-use/scripts/lib/image-provider.mjs +44 -0
- package/skills/media-use/scripts/lib/local-media-search.mjs +32 -0
- package/skills/media-use/scripts/lib/local-models.mjs +318 -0
- package/skills/media-use/scripts/lib/local-models.test.mjs +220 -0
- package/skills/media-use/scripts/lib/local-run.mjs +78 -0
- package/skills/media-use/scripts/lib/local-run.test.mjs +121 -0
- package/skills/media-use/scripts/lib/logo-provider.mjs +244 -0
- package/skills/media-use/scripts/lib/logo-provider.test.mjs +298 -0
- package/skills/media-use/scripts/lib/ltx-video-provider.mjs +94 -0
- package/skills/media-use/scripts/lib/ltx-video-provider.test.mjs +316 -0
- package/skills/media-use/scripts/lib/lut-preset-provider.mjs +271 -0
- package/skills/media-use/scripts/lib/lut-preset-provider.test.mjs +199 -0
- package/skills/media-use/scripts/lib/main-module.mjs +13 -0
- package/skills/media-use/scripts/lib/manifest.d.mts +20 -0
- package/skills/media-use/scripts/lib/manifest.test.mjs +397 -0
- package/skills/media-use/scripts/lib/match.mjs +47 -0
- package/skills/media-use/scripts/lib/media-fetch.test.mjs +197 -0
- package/skills/media-use/scripts/lib/media-home.test.mjs +61 -0
- package/skills/media-use/scripts/lib/media-search.d.mts +9 -0
- package/skills/media-use/scripts/lib/media-search.mjs +107 -0
- package/skills/media-use/scripts/lib/mflux-provider.mjs +128 -0
- package/skills/media-use/scripts/lib/mflux-provider.test.mjs +174 -0
- package/skills/media-use/scripts/lib/misses.mjs +49 -0
- package/skills/media-use/scripts/lib/misses.test.mjs +89 -0
- package/skills/media-use/scripts/lib/npx-sync.mjs +2 -28
- package/skills/media-use/scripts/lib/npx-sync.test.mjs +50 -0
- package/skills/media-use/scripts/lib/openverse.mjs +134 -0
- package/skills/media-use/scripts/lib/openverse.test.mjs +219 -0
- package/skills/media-use/scripts/lib/parakeet-words.test.mjs +44 -0
- package/skills/media-use/scripts/lib/prefs-store.concurrent.test.mjs +173 -0
- package/skills/media-use/scripts/lib/prefs-store.test.mjs +245 -0
- package/skills/media-use/scripts/lib/probe.mjs +46 -0
- package/skills/media-use/scripts/lib/probe.test.mjs +221 -0
- package/skills/media-use/scripts/lib/providers.mjs +5 -0
- package/skills/media-use/scripts/lib/recipe-store.test.mjs +391 -0
- package/skills/media-use/scripts/lib/registry.mjs +214 -0
- package/skills/media-use/scripts/lib/registry.test.mjs +262 -0
- package/skills/media-use/scripts/lib/search.mjs +24 -0
- package/skills/media-use/scripts/lib/search.test.mjs +32 -0
- package/skills/media-use/scripts/lib/sfx-provider.mjs +33 -0
- package/skills/media-use/scripts/lib/specs.mjs +132 -0
- package/skills/media-use/scripts/lib/specs.test.mjs +157 -0
- package/skills/media-use/scripts/lib/stats.mjs +117 -0
- package/skills/media-use/scripts/lib/stats.test.mjs +165 -0
- package/skills/media-use/scripts/lib/svg-sanitize.mjs +21 -0
- package/skills/media-use/scripts/lib/telemetry.test.mjs +437 -53
- package/skills/media-use/scripts/lib/test-home.test.mjs +10 -0
- package/skills/media-use/scripts/lib/transcriptCutFade.test.mjs +33 -0
- package/skills/media-use/scripts/lib/tts-local-provider.mjs +94 -0
- package/skills/media-use/scripts/lib/tts-local-provider.test.mjs +72 -0
- package/skills/media-use/scripts/lib/usage.mjs +57 -0
- package/skills/media-use/scripts/lib/usage.test.mjs +54 -0
- package/skills/media-use/scripts/lib/voice-provider.mjs +57 -0
- package/skills/media-use/scripts/lib/voice-provider.test.mjs +46 -0
- package/skills/media-use/scripts/resolve.mjs +1360 -27
- package/skills/motion-graphics/SKILL.md +1 -1
- package/skills/music-to-video/SKILL.md +1 -1
- package/skills/pr-to-video/SKILL.md +1 -1
- package/skills/product-launch-video/SKILL.md +1 -1
- package/skills/talking-head-recut/SKILL.md +1 -1
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { mkdtempSync, readdirSync, rmSync } from "node:fs";
|
|
3
|
+
import { tmpdir } from "node:os";
|
|
4
|
+
import { join } from "node:path";
|
|
5
|
+
import { test } from "node:test";
|
|
6
|
+
import { fetchMedia, isPublicMediaUrl, readCappedBody } from "./media-fetch.mjs";
|
|
7
|
+
import { freezeUrl } from "./freeze.mjs";
|
|
8
|
+
import { downloadTo } from "../../audio/scripts/lib/heygen.mjs";
|
|
9
|
+
import { synthesizeHeygen } from "../../audio/scripts/lib/tts.mjs";
|
|
10
|
+
import { faviconSearch } from "./logo-provider.mjs";
|
|
11
|
+
|
|
12
|
+
test("blocks private, reserved, alternate-encoded and mapped hosts", () => {
|
|
13
|
+
for (const host of [
|
|
14
|
+
"localhost",
|
|
15
|
+
"localhost.",
|
|
16
|
+
"a.localhost",
|
|
17
|
+
"svc.internal",
|
|
18
|
+
"svc.local",
|
|
19
|
+
"127.1",
|
|
20
|
+
"2130706433",
|
|
21
|
+
"0x7f000001",
|
|
22
|
+
"10.0.0.1",
|
|
23
|
+
"169.254.169.254",
|
|
24
|
+
"100.100.100.200",
|
|
25
|
+
"192.168.1.1",
|
|
26
|
+
"172.31.0.1",
|
|
27
|
+
"224.0.0.1",
|
|
28
|
+
"[::]",
|
|
29
|
+
"[::1]",
|
|
30
|
+
"[::ffff:127.0.0.1]",
|
|
31
|
+
"[::ffff:a00:1]",
|
|
32
|
+
"[fe80::1]",
|
|
33
|
+
"[fd00::1]",
|
|
34
|
+
"[ff02::1]",
|
|
35
|
+
]) {
|
|
36
|
+
assert.equal(isPublicMediaUrl(`https://${host}/asset`), false, host);
|
|
37
|
+
}
|
|
38
|
+
for (const url of [
|
|
39
|
+
"https://cdn.example/asset.cube",
|
|
40
|
+
"http://example.com/clip.mp4",
|
|
41
|
+
"https://8.8.8.8/a",
|
|
42
|
+
"https://[::ffff:8.8.8.8]/a",
|
|
43
|
+
])
|
|
44
|
+
assert.equal(isPublicMediaUrl(url), true, url);
|
|
45
|
+
assert.equal(isPublicMediaUrl("file:///tmp/a.mp4"), false);
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
test("follows public relative redirects manually and forwards cancellation", async () => {
|
|
49
|
+
const calls = [];
|
|
50
|
+
const signal = new AbortController().signal;
|
|
51
|
+
const response = await fetchMedia("https://cdn.example/a", {
|
|
52
|
+
method: "HEAD",
|
|
53
|
+
signal,
|
|
54
|
+
fetchImpl: async (url, options) => {
|
|
55
|
+
calls.push(url);
|
|
56
|
+
assert.equal(options.redirect, "manual");
|
|
57
|
+
assert.equal(options.signal, signal);
|
|
58
|
+
assert.equal(options.method, "HEAD");
|
|
59
|
+
return calls.length === 1
|
|
60
|
+
? new Response(null, { status: 302, headers: { location: "/b" } })
|
|
61
|
+
: new Response("media");
|
|
62
|
+
},
|
|
63
|
+
});
|
|
64
|
+
assert.deepEqual(calls, ["https://cdn.example/a", "https://cdn.example/b"]);
|
|
65
|
+
assert.equal(await response.text(), "media");
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
test("rejects private redirect targets before requesting them and cancels redirect bodies", async () => {
|
|
69
|
+
let calls = 0;
|
|
70
|
+
let cancelled = false;
|
|
71
|
+
await assert.rejects(
|
|
72
|
+
fetchMedia("https://cdn.example/a", {
|
|
73
|
+
fetchImpl: async () => {
|
|
74
|
+
calls++;
|
|
75
|
+
return new Response(
|
|
76
|
+
new ReadableStream({
|
|
77
|
+
cancel() {
|
|
78
|
+
cancelled = true;
|
|
79
|
+
},
|
|
80
|
+
}),
|
|
81
|
+
{
|
|
82
|
+
status: 302,
|
|
83
|
+
headers: { location: "http://169.254.169.254/latest/meta-data" },
|
|
84
|
+
},
|
|
85
|
+
);
|
|
86
|
+
},
|
|
87
|
+
}),
|
|
88
|
+
/blocked/,
|
|
89
|
+
);
|
|
90
|
+
assert.equal(calls, 1);
|
|
91
|
+
assert.equal(cancelled, true);
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
test("bounds redirect loops", async () => {
|
|
95
|
+
let calls = 0;
|
|
96
|
+
await assert.rejects(
|
|
97
|
+
fetchMedia("https://cdn.example/a", {
|
|
98
|
+
fetchImpl: async () => {
|
|
99
|
+
calls++;
|
|
100
|
+
return new Response(null, { status: 302, headers: { location: "/a" } });
|
|
101
|
+
},
|
|
102
|
+
}),
|
|
103
|
+
/redirect limit/,
|
|
104
|
+
);
|
|
105
|
+
assert.equal(calls, 6);
|
|
106
|
+
});
|
|
107
|
+
|
|
108
|
+
test("freeze refuses a public-to-private redirect without writing response bytes", async (t) => {
|
|
109
|
+
const dir = mkdtempSync(join(tmpdir(), "hf-freeze-redirect-"));
|
|
110
|
+
t.mock.method(globalThis, "fetch", async (_url, options) =>
|
|
111
|
+
options?.redirect === "manual"
|
|
112
|
+
? new Response(null, { status: 302, headers: { location: "http://127.0.0.1/private.mp4" } })
|
|
113
|
+
: new Response("private response"),
|
|
114
|
+
);
|
|
115
|
+
try {
|
|
116
|
+
await assert.rejects(freezeUrl("https://cdn.example/a.mp4", join(dir, "out.mp4")), /blocked/);
|
|
117
|
+
assert.deepEqual(readdirSync(dir), []);
|
|
118
|
+
} finally {
|
|
119
|
+
rmSync(dir, { recursive: true, force: true });
|
|
120
|
+
}
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
for (const entry of ["heygen audio", "tts mp3", "tts wav", "favicon"]) {
|
|
124
|
+
test(`${entry} blocks private redirects before saving or transcoding`, async (t) => {
|
|
125
|
+
const dir = mkdtempSync(join(tmpdir(), "hf-provider-redirect-"));
|
|
126
|
+
const requests = [];
|
|
127
|
+
const fetchImpl = t.mock.method(globalThis, "fetch", async (url, options) => {
|
|
128
|
+
requests.push(url);
|
|
129
|
+
return options?.redirect === "manual"
|
|
130
|
+
? new Response(null, {
|
|
131
|
+
status: 302,
|
|
132
|
+
headers: { location: "http://169.254.169.254/private" },
|
|
133
|
+
})
|
|
134
|
+
: new Response(new Uint8Array(600));
|
|
135
|
+
});
|
|
136
|
+
try {
|
|
137
|
+
if (entry === "heygen audio") {
|
|
138
|
+
await assert.rejects(
|
|
139
|
+
downloadTo("https://cdn.example/audio", join(dir, "audio.mp3")),
|
|
140
|
+
/blocked/,
|
|
141
|
+
);
|
|
142
|
+
} else if (entry === "favicon") {
|
|
143
|
+
assert.equal(await faviconSearch("GitHub logo"), null);
|
|
144
|
+
} else {
|
|
145
|
+
const result = await synthesizeHeygen(
|
|
146
|
+
{
|
|
147
|
+
text: "hi",
|
|
148
|
+
voiceId: "v1",
|
|
149
|
+
lang: "en",
|
|
150
|
+
speed: 1,
|
|
151
|
+
wavAbs: join(dir, entry === "tts wav" ? "audio.wav" : "audio.mp3"),
|
|
152
|
+
},
|
|
153
|
+
{
|
|
154
|
+
heygenAuthHeaders: () => ({}),
|
|
155
|
+
heygenJSON: async () => ({ data: { audio_url: "https://cdn.example/audio" } }),
|
|
156
|
+
fetch: fetchImpl,
|
|
157
|
+
transcodeToWav: () => assert.fail("private bytes must not reach ffmpeg"),
|
|
158
|
+
},
|
|
159
|
+
);
|
|
160
|
+
assert.equal(result.ok, false);
|
|
161
|
+
assert.match(result.error, /blocked/);
|
|
162
|
+
}
|
|
163
|
+
assert.equal(requests.length, 1);
|
|
164
|
+
assert.deepEqual(readdirSync(dir), []);
|
|
165
|
+
} finally {
|
|
166
|
+
rmSync(dir, { recursive: true, force: true });
|
|
167
|
+
}
|
|
168
|
+
});
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
function streamed(sizes, contentLength = null) {
|
|
172
|
+
let pulled = 0;
|
|
173
|
+
const body = (async function* () {
|
|
174
|
+
for (const n of sizes) {
|
|
175
|
+
pulled++;
|
|
176
|
+
yield new Uint8Array(n);
|
|
177
|
+
}
|
|
178
|
+
})();
|
|
179
|
+
return { res: { headers: { get: () => contentLength }, body }, pulled: () => pulled };
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
test("readCappedBody refuses a declared oversize body before reading a byte", async () => {
|
|
183
|
+
const { res, pulled } = streamed([10], "101");
|
|
184
|
+
await assert.rejects(readCappedBody(res, 100, "t"), /t: 101 bytes exceeds 100 cap/);
|
|
185
|
+
assert.equal(pulled(), 0);
|
|
186
|
+
});
|
|
187
|
+
|
|
188
|
+
test("readCappedBody aborts a chunked body the moment it crosses the cap", async () => {
|
|
189
|
+
const { res, pulled } = streamed([60, 60, 60]);
|
|
190
|
+
await assert.rejects(readCappedBody(res, 100, "t"), /t: stream exceeds 100 cap/);
|
|
191
|
+
assert.equal(pulled(), 2);
|
|
192
|
+
});
|
|
193
|
+
|
|
194
|
+
test("readCappedBody returns every byte of a body at the cap", async () => {
|
|
195
|
+
const { res } = streamed([60, 40]);
|
|
196
|
+
assert.equal((await readCappedBody(res, 100, "t")).byteLength, 100);
|
|
197
|
+
});
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { spawnSync } from "node:child_process";
|
|
3
|
+
import { existsSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs";
|
|
4
|
+
import { tmpdir } from "node:os";
|
|
5
|
+
import { join } from "node:path";
|
|
6
|
+
import { test } from "node:test";
|
|
7
|
+
|
|
8
|
+
import { cachePut } from "./cache.mjs";
|
|
9
|
+
import { globalMediaDir } from "./media-home.mjs";
|
|
10
|
+
|
|
11
|
+
const HOME = mkdtempSync(join(tmpdir(), "media-home-"));
|
|
12
|
+
process.env.HYPERFRAMES_MEDIA_HOME = HOME;
|
|
13
|
+
|
|
14
|
+
test("the global cache writes into the media home a test points it at", () => {
|
|
15
|
+
const work = mkdtempSync(join(tmpdir(), "media-home-src-"));
|
|
16
|
+
const file = join(work, "clip.wav");
|
|
17
|
+
writeFileSync(file, "media-home regression bytes");
|
|
18
|
+
cachePut(file, { id: "bgm_001", type: "bgm", provenance: { prompt: "media home probe" } });
|
|
19
|
+
const manifest = join(HOME, ".media", "manifest.jsonl");
|
|
20
|
+
assert.ok(existsSync(manifest), "the temp media home holds the global manifest");
|
|
21
|
+
assert.match(readFileSync(manifest, "utf8"), /media home probe/);
|
|
22
|
+
assert.equal(globalMediaDir(), join(HOME, ".media"));
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
// A child process that resolves the media dir, with only the given media-home variables set.
|
|
26
|
+
function runProbe(env) {
|
|
27
|
+
const lib = new URL("./media-home.mjs", import.meta.url).href;
|
|
28
|
+
const childEnv = { ...process.env };
|
|
29
|
+
delete childEnv.HYPERFRAMES_MEDIA_HOME;
|
|
30
|
+
delete childEnv.HYPERFRAMES_MEDIA_HOME_REQUIRED;
|
|
31
|
+
delete childEnv.NODE_TEST_CONTEXT;
|
|
32
|
+
const run = spawnSync(
|
|
33
|
+
process.execPath,
|
|
34
|
+
[
|
|
35
|
+
"--input-type=module",
|
|
36
|
+
"-e",
|
|
37
|
+
`const m = await import(${JSON.stringify(lib)}); console.log("dir=" + m.globalMediaDir());`,
|
|
38
|
+
],
|
|
39
|
+
{ encoding: "utf8", env: { ...childEnv, ...env } },
|
|
40
|
+
);
|
|
41
|
+
return { status: run.status, output: `${run.stdout}${run.stderr}` };
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
test("a repo test run that never points the media home anywhere fails instead of writing", () => {
|
|
45
|
+
const { status, output } = runProbe({ HYPERFRAMES_MEDIA_HOME_REQUIRED: "1" });
|
|
46
|
+
assert.notEqual(status, 0, output);
|
|
47
|
+
assert.match(output, /set HYPERFRAMES_MEDIA_HOME to a temp dir/);
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
// Node sets NODE_TEST_CONTEXT for every project's test runs, so another project whose own tests
|
|
51
|
+
// spawn this CLI must get the person's real library, not the guard.
|
|
52
|
+
test("another project's node --test run still resolves the real media home", () => {
|
|
53
|
+
const home = mkdtempSync(join(tmpdir(), "media-home-user-"));
|
|
54
|
+
const { status, output } = runProbe({
|
|
55
|
+
HOME: home,
|
|
56
|
+
USERPROFILE: home,
|
|
57
|
+
NODE_TEST_CONTEXT: "child-v8",
|
|
58
|
+
});
|
|
59
|
+
assert.equal(status, 0, output);
|
|
60
|
+
assert.ok(output.includes(`dir=${join(home, ".media")}`), output);
|
|
61
|
+
});
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
const STOP = new Set(
|
|
2
|
+
(
|
|
3
|
+
"the a an and or of to in on at is are be it its for with that this as by from into " +
|
|
4
|
+
"one two must not no all over under across while when where which who whom whose they " +
|
|
5
|
+
"them their we our you your he she his her but if then than so such can may might will " +
|
|
6
|
+
"would should each other another same both few more most some any every"
|
|
7
|
+
).split(" "),
|
|
8
|
+
);
|
|
9
|
+
|
|
10
|
+
const PLURAL_RULES = [
|
|
11
|
+
[/([^aeiou])ies$/, "$1y"],
|
|
12
|
+
[/(ch|sh|ss|x)es$/, "$1"],
|
|
13
|
+
[/(ss|us|is)$/, "$&"],
|
|
14
|
+
[/s$/, ""],
|
|
15
|
+
];
|
|
16
|
+
const STRONG_FIELD_WEIGHT = 3;
|
|
17
|
+
const INFERRED_TOKEN_WEIGHT = 0.35;
|
|
18
|
+
|
|
19
|
+
function singularize(word) {
|
|
20
|
+
if (word.length <= 3) return word;
|
|
21
|
+
for (const [pattern, replacement] of PLURAL_RULES) {
|
|
22
|
+
if (pattern.test(word)) return word.replace(pattern, replacement);
|
|
23
|
+
}
|
|
24
|
+
return word;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
function tokenize(text) {
|
|
28
|
+
const words =
|
|
29
|
+
String(text)
|
|
30
|
+
.toLowerCase()
|
|
31
|
+
.match(/[a-z]+/g) || [];
|
|
32
|
+
return words.filter((word) => word.length > 2 && !STOP.has(word)).map(singularize);
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export function rankMediaRows(query, rows) {
|
|
36
|
+
const asked = tokenize(query);
|
|
37
|
+
const normalizedLiteralQuery = String(query).trim().toLowerCase();
|
|
38
|
+
const normalizedQuery = asked.join(" ");
|
|
39
|
+
const want = new Map(asked.map((token) => [token, 1]));
|
|
40
|
+
if (want.size === 0) return [];
|
|
41
|
+
|
|
42
|
+
const parsed = rows.map((row) => {
|
|
43
|
+
const strongTokens = new Set(tokenize(`${row.id} ${row.title}`));
|
|
44
|
+
const allTokens = new Set([
|
|
45
|
+
...strongTokens,
|
|
46
|
+
...tokenize(`${row.description} ${row.tags.join(" ")} ${row.kind}`),
|
|
47
|
+
]);
|
|
48
|
+
return {
|
|
49
|
+
row,
|
|
50
|
+
strongTokens,
|
|
51
|
+
allTokens,
|
|
52
|
+
literalId: row.id.trim().toLowerCase() === normalizedLiteralQuery,
|
|
53
|
+
stemmedId: tokenize(row.id).join(" ") === normalizedQuery,
|
|
54
|
+
};
|
|
55
|
+
});
|
|
56
|
+
const vocabulary = new Set(parsed.flatMap(({ allTokens }) => [...allTokens]));
|
|
57
|
+
for (const token of asked) {
|
|
58
|
+
if (token.length < 6) continue;
|
|
59
|
+
for (let cut = 3; cut <= token.length - 3; cut++) {
|
|
60
|
+
const head = token.slice(0, cut);
|
|
61
|
+
const tail = token.slice(cut);
|
|
62
|
+
if (vocabulary.has(head) && vocabulary.has(tail)) {
|
|
63
|
+
want.set(head, INFERRED_TOKEN_WEIGHT);
|
|
64
|
+
want.set(tail, INFERRED_TOKEN_WEIGHT);
|
|
65
|
+
break;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
for (let index = 0; index < asked.length - 1; index++) {
|
|
70
|
+
const joined = `${asked[index]}${asked[index + 1]}`;
|
|
71
|
+
if (vocabulary.has(joined)) want.set(joined, INFERRED_TOKEN_WEIGHT);
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
const idf = new Map();
|
|
75
|
+
for (const token of want.keys()) {
|
|
76
|
+
const documentFrequency = parsed.filter(({ allTokens }) => allTokens.has(token)).length;
|
|
77
|
+
idf.set(token, Math.log((parsed.length + 1) / (documentFrequency + 1)) + 1);
|
|
78
|
+
}
|
|
79
|
+
return parsed
|
|
80
|
+
.map(({ row, strongTokens, allTokens, literalId, stemmedId }) => {
|
|
81
|
+
let score = 0;
|
|
82
|
+
for (const [token, asking] of want) {
|
|
83
|
+
const weight = (idf.get(token) || 1) * asking;
|
|
84
|
+
if (strongTokens.has(token)) score += STRONG_FIELD_WEIGHT * weight;
|
|
85
|
+
else if (allTokens.has(token)) score += weight;
|
|
86
|
+
}
|
|
87
|
+
return { row, score, literalId, stemmedId };
|
|
88
|
+
})
|
|
89
|
+
.filter(({ score }) => score > 0)
|
|
90
|
+
.sort(
|
|
91
|
+
(a, b) =>
|
|
92
|
+
Number(b.literalId) - Number(a.literalId) ||
|
|
93
|
+
Number(b.stemmedId) - Number(a.stemmedId) ||
|
|
94
|
+
b.score - a.score,
|
|
95
|
+
)
|
|
96
|
+
.map(({ row }) => row);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
export async function rankMediaRowsWithVectors(query, rows, semanticRanking) {
|
|
100
|
+
const words = rankMediaRows(query, rows);
|
|
101
|
+
if (words.length > 0) return { rows: words, tier: "words" };
|
|
102
|
+
if (!semanticRanking) return { rows: [], tier: "words" };
|
|
103
|
+
const semantic = await semanticRanking(query, rows);
|
|
104
|
+
return semantic
|
|
105
|
+
? { rows: semantic.map(({ row }) => row), tier: "on-device" }
|
|
106
|
+
: { rows: [], tier: "words" };
|
|
107
|
+
}
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
import { execFileSync } from "node:child_process";
|
|
2
|
+
import { existsSync, unlinkSync } from "node:fs";
|
|
3
|
+
import { tmpdir } from "node:os";
|
|
4
|
+
import { join } from "node:path";
|
|
5
|
+
import { describeDownload, probeSpecs } from "./specs.mjs";
|
|
6
|
+
import { buildArgv, selectModelLadder } from "./local-models.mjs";
|
|
7
|
+
|
|
8
|
+
// Local image generation via mflux (FLUX-on-MLX), the Mac-native runner.
|
|
9
|
+
// Spec-gated: selectModelLadder("imagegen", specs) returns every FLUX-class
|
|
10
|
+
// model the machine's AVAILABLE RAM can actually run (medium FLUX-schnell
|
|
11
|
+
// --low-ram on ~24GB, up to Qwen-Image on 64GB+), best first. When nothing
|
|
12
|
+
// local fits, or no fitting tier can actually run here, this returns null so
|
|
13
|
+
// the registry falls through to the codex image upsell.
|
|
14
|
+
//
|
|
15
|
+
// The official FLUX repos are HF-gated, so the model entries point --path at
|
|
16
|
+
// non-gated community 4-bit re-uploads; the repo is resolved to a local snapshot
|
|
17
|
+
// (hf download, idempotent) because a bare repo id breaks mlx unflatten.
|
|
18
|
+
|
|
19
|
+
// Resolve an HF repo to its local snapshot dir. `hf download` is idempotent and
|
|
20
|
+
// prints the snapshot path as its last line.
|
|
21
|
+
function resolveSnapshot(repo, execFn, pathExists) {
|
|
22
|
+
const out = execFn("hf", ["download", repo], {
|
|
23
|
+
encoding: "utf8",
|
|
24
|
+
timeout: 1_800_000,
|
|
25
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
26
|
+
});
|
|
27
|
+
const path = out?.trim().split(/\r?\n/).pop()?.trim();
|
|
28
|
+
return path && pathExists(path) ? path : null;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export async function mfluxImageGenerate(
|
|
32
|
+
intent,
|
|
33
|
+
ctx,
|
|
34
|
+
execFn = execFileSync,
|
|
35
|
+
pathExists = existsSync,
|
|
36
|
+
unlinkFn = unlinkSync,
|
|
37
|
+
) {
|
|
38
|
+
const specs = ctx?.specs || probeSpecs();
|
|
39
|
+
const ladder = selectModelLadder("imagegen", specs, { preferTier: ctx?.preferTier });
|
|
40
|
+
if (!ladder.length) return null; // no local model fits -> codex upsell/fallback
|
|
41
|
+
|
|
42
|
+
// Each attempt mints its own timestamped output path, so a partial artifact
|
|
43
|
+
// from a failed tier is orphaned rather than overwritten - and a lower tier
|
|
44
|
+
// then succeeding hides it. Discard it before demoting. Best-effort: a
|
|
45
|
+
// partial we cannot remove must never mask the real failure.
|
|
46
|
+
const discardPartial = (path) => {
|
|
47
|
+
try {
|
|
48
|
+
if (pathExists(path)) unlinkFn(path);
|
|
49
|
+
} catch {
|
|
50
|
+
// nothing actionable: the generate failure below is the real story
|
|
51
|
+
}
|
|
52
|
+
};
|
|
53
|
+
|
|
54
|
+
// Best tier first, demoting past any tier that cannot run here (runner off
|
|
55
|
+
// PATH, a snapshot that won't download, an OOM) rather than failing local
|
|
56
|
+
// image gen outright. Every demotion is reported, so a quietly smaller model
|
|
57
|
+
// is never mistaken for the tier the machine nominally qualified for.
|
|
58
|
+
for (const model of ladder) {
|
|
59
|
+
const bin = model.invoke.trim().split(/\s+/)[0];
|
|
60
|
+
// Not installed? Surface the exact enable-command (before the model
|
|
61
|
+
// download) so the agent learns the free local path is available instead of
|
|
62
|
+
// silently taking the codex upsell.
|
|
63
|
+
try {
|
|
64
|
+
execFn("which", [bin], { stdio: ["ignore", "ignore", "ignore"] });
|
|
65
|
+
} catch {
|
|
66
|
+
console.error(
|
|
67
|
+
`media-use: local image gen not enabled (\`${bin}\` not on PATH). Install for free on-device FLUX: ${model.install}. Heads up: ${model.id} ${describeDownload(model.sizeMB)}.`,
|
|
68
|
+
);
|
|
69
|
+
continue;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
const outPath = join(tmpdir(), `media-use-mflux-${process.pid}-${Date.now()}.png`);
|
|
73
|
+
const vars = {
|
|
74
|
+
prompt: intent,
|
|
75
|
+
w: ctx?.width || 512,
|
|
76
|
+
h: ctx?.height || 512,
|
|
77
|
+
seed: ctx?.seed ?? 42,
|
|
78
|
+
out: outPath,
|
|
79
|
+
};
|
|
80
|
+
if (model.repo && model.invoke.includes("{model_path}")) {
|
|
81
|
+
let snap = null;
|
|
82
|
+
let why = `could not resolve a local snapshot of ${model.repo}`;
|
|
83
|
+
try {
|
|
84
|
+
snap = resolveSnapshot(model.repo, execFn, pathExists);
|
|
85
|
+
} catch (err) {
|
|
86
|
+
why = `hf download failed: ${err.stderr?.toString().trim().slice(-200) || err.message}`;
|
|
87
|
+
}
|
|
88
|
+
if (!snap) {
|
|
89
|
+
console.error(`media-use: local image gen (${model.id}): ${why}`);
|
|
90
|
+
continue;
|
|
91
|
+
}
|
|
92
|
+
vars.model_path = snap;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
const argv = buildArgv(model.invoke, vars);
|
|
96
|
+
argv.shift(); // drop the bin (already validated)
|
|
97
|
+
try {
|
|
98
|
+
execFn(bin, argv, {
|
|
99
|
+
encoding: "utf8",
|
|
100
|
+
timeout: 1_800_000,
|
|
101
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
102
|
+
});
|
|
103
|
+
} catch (err) {
|
|
104
|
+
discardPartial(outPath);
|
|
105
|
+
console.error(
|
|
106
|
+
`media-use: local image gen (${model.id}) failed: ${err.stderr?.toString().trim().slice(-200) || err.message}`,
|
|
107
|
+
);
|
|
108
|
+
continue;
|
|
109
|
+
}
|
|
110
|
+
if (!pathExists(outPath)) {
|
|
111
|
+
console.error(
|
|
112
|
+
`media-use: local image gen (${model.id}) exited cleanly but wrote no output file`,
|
|
113
|
+
);
|
|
114
|
+
continue;
|
|
115
|
+
}
|
|
116
|
+
return {
|
|
117
|
+
localPath: outPath,
|
|
118
|
+
ext: ".png",
|
|
119
|
+
source: "generated",
|
|
120
|
+
metadata: {
|
|
121
|
+
description: intent,
|
|
122
|
+
provider: `mflux.${model.id}`,
|
|
123
|
+
provenance: { model: model.id, tier: model.tier, prompt: intent },
|
|
124
|
+
},
|
|
125
|
+
};
|
|
126
|
+
}
|
|
127
|
+
return null;
|
|
128
|
+
}
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
import { test } from "node:test";
|
|
2
|
+
import assert from "node:assert/strict";
|
|
3
|
+
import { mfluxImageGenerate } from "./mflux-provider.mjs";
|
|
4
|
+
|
|
5
|
+
// 40GB clears the 32GB klein tier and the 8GB schnell tier below it; the 64GB
|
|
6
|
+
// qwen tier stays out of reach. Two rungs is what makes demotion observable.
|
|
7
|
+
const bothTiersSpecs = { availableRamMB: 40000, gpu: { present: true } };
|
|
8
|
+
const SNAPSHOT = "/tmp/hf-snapshot";
|
|
9
|
+
|
|
10
|
+
// exec stub covering all three shells-out mflux does: the PATH probe, the
|
|
11
|
+
// idempotent `hf download`, and the generate itself.
|
|
12
|
+
function stubExec({ failGenerateFor = [], failWhichFor = [] } = {}) {
|
|
13
|
+
const calls = [];
|
|
14
|
+
const exec = (...call) => {
|
|
15
|
+
calls.push(call);
|
|
16
|
+
const [bin, argv] = call;
|
|
17
|
+
if (bin === "which") {
|
|
18
|
+
if (failWhichFor.includes(argv[0])) throw new Error("not found");
|
|
19
|
+
return "";
|
|
20
|
+
}
|
|
21
|
+
if (bin === "hf") return `Fetching 6 files...\n${SNAPSHOT}\n`;
|
|
22
|
+
if (failGenerateFor.some((id) => argv.join(" ").includes(id))) {
|
|
23
|
+
const err = new Error("exit 1");
|
|
24
|
+
err.stderr = "mlx.core.metal: out of memory";
|
|
25
|
+
throw err;
|
|
26
|
+
}
|
|
27
|
+
return "";
|
|
28
|
+
};
|
|
29
|
+
return { calls, exec };
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
const generateCalls = (calls) =>
|
|
33
|
+
calls.filter(([bin]) => bin !== "which" && bin !== "hf").map(([, argv]) => argv.join(" "));
|
|
34
|
+
|
|
35
|
+
test("no local model fits: falls through to the upsell without shelling out", async () => {
|
|
36
|
+
const { calls, exec } = stubExec();
|
|
37
|
+
|
|
38
|
+
const result = await mfluxImageGenerate(
|
|
39
|
+
"a red bicycle",
|
|
40
|
+
{ specs: { availableRamMB: 100, gpu: { present: true } } },
|
|
41
|
+
exec,
|
|
42
|
+
() => true,
|
|
43
|
+
);
|
|
44
|
+
|
|
45
|
+
assert.equal(result, null);
|
|
46
|
+
assert.deepEqual(calls, []);
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
test("a top tier that cannot run demotes to the next fitting tier", async (t) => {
|
|
50
|
+
const errors = [];
|
|
51
|
+
t.mock.method(console, "error", (message) => errors.push(message));
|
|
52
|
+
const { calls, exec } = stubExec({ failGenerateFor: ["flux2-klein-4b"] });
|
|
53
|
+
|
|
54
|
+
const result = await mfluxImageGenerate(
|
|
55
|
+
"a red bicycle",
|
|
56
|
+
{ specs: bothTiersSpecs },
|
|
57
|
+
exec,
|
|
58
|
+
() => true,
|
|
59
|
+
);
|
|
60
|
+
|
|
61
|
+
assert.ok(result, "the schnell tier still produced an image");
|
|
62
|
+
assert.equal(result.metadata.provider, "mflux.flux-schnell-mflux-q4");
|
|
63
|
+
const generated = generateCalls(calls);
|
|
64
|
+
assert.equal(generated.length, 2, "klein attempted first, then schnell");
|
|
65
|
+
assert.match(generated[0], /flux2-klein-4b/);
|
|
66
|
+
assert.match(generated[1], /--model schnell/);
|
|
67
|
+
assert.equal(errors.length, 1, "the demotion is reported, not silent");
|
|
68
|
+
assert.match(errors[0], /flux2-klein-mflux-q4\) failed/);
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
test("a snapshot that will not resolve demotes rather than failing outright", async (t) => {
|
|
72
|
+
const errors = [];
|
|
73
|
+
t.mock.method(console, "error", (message) => errors.push(message));
|
|
74
|
+
const calls = [];
|
|
75
|
+
const exec = (...call) => {
|
|
76
|
+
calls.push(call);
|
|
77
|
+
const [bin, argv] = call;
|
|
78
|
+
if (bin === "which") return "";
|
|
79
|
+
// klein's weights won't download; schnell's do
|
|
80
|
+
if (bin === "hf") {
|
|
81
|
+
if (argv[1].includes("FLUX.2-klein")) throw new Error("403 Forbidden");
|
|
82
|
+
return `${SNAPSHOT}\n`;
|
|
83
|
+
}
|
|
84
|
+
return "";
|
|
85
|
+
};
|
|
86
|
+
|
|
87
|
+
const result = await mfluxImageGenerate(
|
|
88
|
+
"a red bicycle",
|
|
89
|
+
{ specs: bothTiersSpecs },
|
|
90
|
+
exec,
|
|
91
|
+
() => true,
|
|
92
|
+
);
|
|
93
|
+
|
|
94
|
+
assert.ok(result, "demoted past the ungettable weights");
|
|
95
|
+
assert.equal(result.metadata.provider, "mflux.flux-schnell-mflux-q4");
|
|
96
|
+
assert.equal(errors.length, 1);
|
|
97
|
+
assert.match(errors[0], /hf download failed/);
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
test("runner missing from PATH reports the install hint per tier and returns null", async (t) => {
|
|
101
|
+
const errors = [];
|
|
102
|
+
t.mock.method(console, "error", (message) => errors.push(message));
|
|
103
|
+
const { exec } = stubExec({ failWhichFor: ["mflux-generate"] });
|
|
104
|
+
|
|
105
|
+
const result = await mfluxImageGenerate(
|
|
106
|
+
"a red bicycle",
|
|
107
|
+
{ specs: bothTiersSpecs },
|
|
108
|
+
exec,
|
|
109
|
+
() => true,
|
|
110
|
+
);
|
|
111
|
+
|
|
112
|
+
assert.equal(result, null);
|
|
113
|
+
assert.equal(errors.length, 2, "both fitting tiers reported");
|
|
114
|
+
assert.match(errors[0], /uv pip install mflux/);
|
|
115
|
+
// each hint states that tier's download cost before the user commits
|
|
116
|
+
assert.match(errors[0], /GB of weights to/);
|
|
117
|
+
assert.match(errors[1], /GB of weights to/);
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
// Same per-attempt temp path, same orphaning risk as the LTX provider: a
|
|
121
|
+
// partial png from a failed tier must not survive a lower tier succeeding.
|
|
122
|
+
const outputOf = (argv) => argv[argv.indexOf("--output") + 1];
|
|
123
|
+
const attemptedOutputs = (calls) =>
|
|
124
|
+
calls.filter(([bin]) => bin !== "which" && bin !== "hf").map(([, argv]) => outputOf(argv));
|
|
125
|
+
|
|
126
|
+
test("a failed attempt's partial output is discarded before demoting", async (t) => {
|
|
127
|
+
t.mock.method(console, "error", () => {});
|
|
128
|
+
const unlinked = [];
|
|
129
|
+
const { calls, exec } = stubExec({ failGenerateFor: ["flux2-klein-4b"] });
|
|
130
|
+
|
|
131
|
+
const result = await mfluxImageGenerate(
|
|
132
|
+
"a red bicycle",
|
|
133
|
+
{ specs: bothTiersSpecs },
|
|
134
|
+
exec,
|
|
135
|
+
() => true,
|
|
136
|
+
(path) => unlinked.push(path),
|
|
137
|
+
);
|
|
138
|
+
|
|
139
|
+
assert.ok(result, "the schnell tier still produced an image");
|
|
140
|
+
assert.deepEqual(unlinked, [attemptedOutputs(calls)[0]], "the failed klein partial is removed");
|
|
141
|
+
});
|
|
142
|
+
|
|
143
|
+
test("every tier failing discards every partial, one per attempt", async (t) => {
|
|
144
|
+
t.mock.method(console, "error", () => {});
|
|
145
|
+
const unlinked = [];
|
|
146
|
+
const { calls, exec } = stubExec({ failGenerateFor: ["flux2-klein-4b", "schnell"] });
|
|
147
|
+
|
|
148
|
+
const result = await mfluxImageGenerate(
|
|
149
|
+
"a red bicycle",
|
|
150
|
+
{ specs: bothTiersSpecs },
|
|
151
|
+
exec,
|
|
152
|
+
() => true,
|
|
153
|
+
(path) => unlinked.push(path),
|
|
154
|
+
);
|
|
155
|
+
|
|
156
|
+
assert.equal(result, null);
|
|
157
|
+
assert.deepEqual(unlinked, attemptedOutputs(calls), "nothing is left behind");
|
|
158
|
+
});
|
|
159
|
+
|
|
160
|
+
test("a successful generation is never discarded", async () => {
|
|
161
|
+
const unlinked = [];
|
|
162
|
+
const { exec } = stubExec();
|
|
163
|
+
|
|
164
|
+
const result = await mfluxImageGenerate(
|
|
165
|
+
"a red bicycle",
|
|
166
|
+
{ specs: bothTiersSpecs },
|
|
167
|
+
exec,
|
|
168
|
+
() => true,
|
|
169
|
+
(path) => unlinked.push(path),
|
|
170
|
+
);
|
|
171
|
+
|
|
172
|
+
assert.ok(result);
|
|
173
|
+
assert.deepEqual(unlinked, [], "the returned artifact must survive");
|
|
174
|
+
});
|