@viralspin/mcp 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +11 -1
- package/dist/client.js +4 -4
- package/dist/index.js +7 -0
- package/dist/local-render/cwd.js +29 -0
- package/dist/local-render/media-fix.js +71 -0
- package/dist/local-render/render.js +25 -3
- package/dist/local-render/startup.js +9 -9
- package/dist/local-render/tokens.js +4 -0
- package/dist/local-render/watch.js +49 -13
- package/dist/organize-watch.js +551 -0
- package/dist/organizer/edit.js +148 -0
- package/dist/organizer/images.js +163 -0
- package/dist/organizer/plan.js +204 -0
- package/dist/organizer/scan.js +331 -0
- package/dist/organizer/topics.js +204 -0
- package/dist/tools/organize.js +171 -0
- package/package.json +46 -42
|
@@ -0,0 +1,331 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
36
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
37
|
+
};
|
|
38
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
39
|
+
exports.pairLive = pairLive;
|
|
40
|
+
exports.walkAsync = walkAsync;
|
|
41
|
+
exports.walk = walk;
|
|
42
|
+
exports.organizeScan = organizeScan;
|
|
43
|
+
exports.makeThumbsFor = makeThumbsFor;
|
|
44
|
+
/**
|
|
45
|
+
* Scan a folder tree (read-only) and propose how to organize it. Reports progress as it goes (the web animation follows it). Nothing is moved here:
|
|
46
|
+
* applying is a separate, confirmed step (plan.ts applyMoves).
|
|
47
|
+
*/
|
|
48
|
+
const crypto_1 = __importDefault(require("crypto"));
|
|
49
|
+
const fs_1 = __importDefault(require("fs"));
|
|
50
|
+
const path_1 = __importDefault(require("path"));
|
|
51
|
+
const url_1 = require("url");
|
|
52
|
+
const plan_1 = require("./plan");
|
|
53
|
+
const images_1 = require("./images");
|
|
54
|
+
const topics_1 = require("./topics");
|
|
55
|
+
/**
|
|
56
|
+
* Live Photos: a still and a short clip with the same name in the same folder (IMG_1234.HEIC + IMG_1234.MOV) are ONE moment. The clip rides on the still
|
|
57
|
+
* (`live`) instead of counting as a second, separate video.
|
|
58
|
+
*/
|
|
59
|
+
function pairLive(items) {
|
|
60
|
+
const key = (it) => it.rel.slice(0, it.rel.length - path_1.default.extname(it.rel).length).toLowerCase();
|
|
61
|
+
const stills = new Map(items.filter((it) => it.kind === "image").map((it) => [key(it), it]));
|
|
62
|
+
const out = [];
|
|
63
|
+
for (const it of items) {
|
|
64
|
+
const still = it.kind === "video" ? stills.get(key(it)) : undefined;
|
|
65
|
+
if (still && it.size < 40000000) {
|
|
66
|
+
still.live = it.rel;
|
|
67
|
+
continue;
|
|
68
|
+
} // a Live Photo clip is a few seconds; a long video with the same name is its own
|
|
69
|
+
out.push(it);
|
|
70
|
+
}
|
|
71
|
+
return out;
|
|
72
|
+
}
|
|
73
|
+
/** walk(), without blocking and with the count so far every few hundred files: a big folder takes a minute to list. */
|
|
74
|
+
async function walkAsync(root, max, onCount) {
|
|
75
|
+
const out = [];
|
|
76
|
+
const stack = [root];
|
|
77
|
+
let told = 0;
|
|
78
|
+
while (stack.length && out.length < max) {
|
|
79
|
+
const dir = stack.pop();
|
|
80
|
+
let ents;
|
|
81
|
+
try {
|
|
82
|
+
ents = await fs_1.default.promises.readdir(dir, { withFileTypes: true });
|
|
83
|
+
}
|
|
84
|
+
catch {
|
|
85
|
+
continue;
|
|
86
|
+
}
|
|
87
|
+
const files = [];
|
|
88
|
+
for (const e of ents) {
|
|
89
|
+
const p = path_1.default.join(dir, e.name);
|
|
90
|
+
if (e.isDirectory()) {
|
|
91
|
+
if (e.name !== plan_1.ORGANIZED_DIR && !e.name.startsWith("."))
|
|
92
|
+
stack.push(p);
|
|
93
|
+
continue;
|
|
94
|
+
}
|
|
95
|
+
const kind = (0, plan_1.kindOf)(e.name);
|
|
96
|
+
if (kind)
|
|
97
|
+
files.push([p, kind]);
|
|
98
|
+
}
|
|
99
|
+
for (let k = 0; k < files.length && out.length < max; k += 64) {
|
|
100
|
+
const sts = await Promise.all(files.slice(k, k + 64).map(([p]) => fs_1.default.promises.stat(p).catch(() => null)));
|
|
101
|
+
sts.forEach((st, j) => { const [p, kind] = files[k + j]; if (st && out.length < max)
|
|
102
|
+
out.push({ rel: path_1.default.relative(root, p), kind, size: st.size, mtime: st.mtimeMs }); });
|
|
103
|
+
if (out.length - told >= 250) {
|
|
104
|
+
told = out.length;
|
|
105
|
+
onCount(told);
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
return out.sort((a, b) => a.rel.localeCompare(b.rel));
|
|
110
|
+
}
|
|
111
|
+
function walk(root, max = 10000) {
|
|
112
|
+
const out = [];
|
|
113
|
+
const stack = [root];
|
|
114
|
+
while (stack.length && out.length < max) {
|
|
115
|
+
const dir = stack.pop();
|
|
116
|
+
let ents;
|
|
117
|
+
try {
|
|
118
|
+
ents = fs_1.default.readdirSync(dir, { withFileTypes: true });
|
|
119
|
+
}
|
|
120
|
+
catch {
|
|
121
|
+
continue;
|
|
122
|
+
}
|
|
123
|
+
for (const e of ents) {
|
|
124
|
+
const p = path_1.default.join(dir, e.name);
|
|
125
|
+
if (e.isDirectory()) {
|
|
126
|
+
if (e.name !== plan_1.ORGANIZED_DIR && !e.name.startsWith("."))
|
|
127
|
+
stack.push(p);
|
|
128
|
+
continue;
|
|
129
|
+
}
|
|
130
|
+
const kind = (0, plan_1.kindOf)(e.name);
|
|
131
|
+
if (!kind || out.length >= max)
|
|
132
|
+
continue;
|
|
133
|
+
try {
|
|
134
|
+
const st = fs_1.default.statSync(p);
|
|
135
|
+
out.push({ rel: path_1.default.relative(root, p), kind, size: st.size, mtime: st.mtimeMs });
|
|
136
|
+
}
|
|
137
|
+
catch { /* unreadable */ }
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
return out.sort((a, b) => a.rel.localeCompare(b.rel));
|
|
141
|
+
}
|
|
142
|
+
/**
|
|
143
|
+
* An exact-copy fingerprint without reading whole files: the size plus the first, middle and last 256 KB. Only files of the same size are compared,
|
|
144
|
+
* and two different photos or clips that also agree on those 768 KB do not happen in a camera roll; hashing every byte of 2,700 phone videos took
|
|
145
|
+
* hours (2026-10-31).
|
|
146
|
+
*/
|
|
147
|
+
const SAMPLE = 256 * 1024;
|
|
148
|
+
const sha1 = async (file) => {
|
|
149
|
+
const fh = await fs_1.default.promises.open(file, "r");
|
|
150
|
+
try {
|
|
151
|
+
const { size } = await fh.stat();
|
|
152
|
+
const h = crypto_1.default.createHash("sha1").update(String(size));
|
|
153
|
+
for (const at of size <= SAMPLE * 3 ? [0] : [0, Math.floor(size / 2 - SAMPLE / 2), size - SAMPLE]) {
|
|
154
|
+
const buf = Buffer.alloc(Math.min(size <= SAMPLE * 3 ? size : SAMPLE, size));
|
|
155
|
+
const { bytesRead } = await fh.read(buf, 0, buf.length, at);
|
|
156
|
+
h.update(buf.subarray(0, bytesRead));
|
|
157
|
+
}
|
|
158
|
+
return h.digest("hex");
|
|
159
|
+
}
|
|
160
|
+
finally {
|
|
161
|
+
await fh.close();
|
|
162
|
+
}
|
|
163
|
+
};
|
|
164
|
+
async function pool(n, list, fn) {
|
|
165
|
+
let k = 0;
|
|
166
|
+
await Promise.all(Array.from({ length: Math.min(n, list.length) }, async () => { while (k < list.length) {
|
|
167
|
+
const i = k++;
|
|
168
|
+
await fn(list[i], i);
|
|
169
|
+
} }));
|
|
170
|
+
}
|
|
171
|
+
/** The local vision model lives in the engine (not in the npm package); without it pictures stay untagged and topic buckets fall back to "Unsorted". */
|
|
172
|
+
async function visionAnalyzer() {
|
|
173
|
+
try {
|
|
174
|
+
const { ENGINE_DIR } = await Promise.resolve().then(() => __importStar(require("../engine")));
|
|
175
|
+
const url = (0, url_1.pathToFileURL)(path_1.default.join(ENGINE_DIR, "gemini.mjs")).href;
|
|
176
|
+
return ((await new Function("u", "return import(u)")(url)).analyzeFrame) ?? null;
|
|
177
|
+
}
|
|
178
|
+
catch {
|
|
179
|
+
return null;
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
async function organizeScan(root, o = {}) {
|
|
183
|
+
const { by: asked = "folder", vision = false, thumbs = "", max = 100000, labels: custom = [], onProgress = () => { } } = o;
|
|
184
|
+
if (!fs_1.default.existsSync(root) || !fs_1.default.statSync(root).isDirectory())
|
|
185
|
+
throw new Error(`folder not found: ${root}`);
|
|
186
|
+
const items = pairLive(await walkAsync(root, max, (n) => onProgress({ phase: "counting", done: n })));
|
|
187
|
+
onProgress({ phase: "found", total: items.length });
|
|
188
|
+
const imgs = items.map((_, i) => i).filter((i) => items[i].kind === "image");
|
|
189
|
+
const t0 = Date.now();
|
|
190
|
+
let done = 0;
|
|
191
|
+
await pool(6, imgs, async (i) => {
|
|
192
|
+
items[i].dhash = (await (0, images_1.dhash)(path_1.default.join(root, items[i].rel))) || undefined;
|
|
193
|
+
items[i].month = (await (0, images_1.takenMonth)(path_1.default.join(root, items[i].rel))) || undefined;
|
|
194
|
+
// the preview now, while the picture is being read anyway: the web page builds its pile from these as they land
|
|
195
|
+
if (thumbs) {
|
|
196
|
+
try {
|
|
197
|
+
fs_1.default.mkdirSync(thumbs, { recursive: true });
|
|
198
|
+
const dst = path_1.default.join(thumbs, `${i}.jpg`);
|
|
199
|
+
if (await (0, images_1.makeThumb)(path_1.default.join(root, items[i].rel), "image", dst)) {
|
|
200
|
+
items[i].thumb = dst;
|
|
201
|
+
o.onThumb?.(dst);
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
catch { /* the thumbs pass tries again */ }
|
|
205
|
+
}
|
|
206
|
+
if (++done % 10 === 0 || done === imgs.length)
|
|
207
|
+
onProgress({ phase: "reading", done, total: imgs.length, etaMs: Math.round(((Date.now() - t0) / done) * (imgs.length - done)) });
|
|
208
|
+
});
|
|
209
|
+
// exact copies after the pictures are read (it reads whole files, slow on big videos; doing it first left the page with nothing to show):
|
|
210
|
+
// a content hash only where two files share a size
|
|
211
|
+
const bySize = new Map();
|
|
212
|
+
items.forEach((it, i) => bySize.set(it.size, [...(bySize.get(it.size) || []), i]));
|
|
213
|
+
const same = [...bySize.values()].filter((v) => v.length > 1).flat();
|
|
214
|
+
let h = 0;
|
|
215
|
+
const th = Date.now();
|
|
216
|
+
await pool(4, same, async (i) => { try {
|
|
217
|
+
items[i].hash = await sha1(path_1.default.join(root, items[i].rel));
|
|
218
|
+
}
|
|
219
|
+
catch { /* skip */ } if (++h % 25 === 0 || h === same.length)
|
|
220
|
+
onProgress({ phase: "hashing", done: h, total: same.length, etaMs: Math.round(((Date.now() - th) / h) * (same.length - h)) }); });
|
|
221
|
+
for (const it of items)
|
|
222
|
+
if (!it.month) {
|
|
223
|
+
const d = new Date(it.mtime);
|
|
224
|
+
it.month = `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, "0")}`;
|
|
225
|
+
}
|
|
226
|
+
(0, plan_1.markDuplicates)(items);
|
|
227
|
+
// topic buckets: the on-device CLIP model first (private, no key), the engine's vision model when that is all there is, else by folder
|
|
228
|
+
let by = asked;
|
|
229
|
+
if (asked === "topic") {
|
|
230
|
+
const todo = items.map((_, i) => i).filter((i) => items[i].dupOf === undefined);
|
|
231
|
+
const set = (0, topics_1.labelSet)(custom);
|
|
232
|
+
const clf = await (0, topics_1.labeller)(set);
|
|
233
|
+
if (clf) {
|
|
234
|
+
const t2 = Date.now();
|
|
235
|
+
let v = 0;
|
|
236
|
+
const found = items.map(() => undefined);
|
|
237
|
+
// the card a picture will most likely land on (its label's group, or the creator's own label), for the page's live deal
|
|
238
|
+
const card = new Map(set.names.map((nm, j) => [nm, set.custom.has(nm) ? nm : set.groups[j]]));
|
|
239
|
+
onProgress({ phase: "looking", done: 0, total: todo.length });
|
|
240
|
+
await pool(2, todo, async (i) => {
|
|
241
|
+
try {
|
|
242
|
+
const rgb = await (0, images_1.rgb224)(path_1.default.join(root, items[i].rel), items[i].kind);
|
|
243
|
+
if (rgb)
|
|
244
|
+
found[i] = await clf(rgb, 224, 224);
|
|
245
|
+
}
|
|
246
|
+
catch { /* leave it for Other */ }
|
|
247
|
+
if (!found[i])
|
|
248
|
+
found[i] = { tags: [], best: null };
|
|
249
|
+
const best = found[i]?.best;
|
|
250
|
+
o.onLabel?.(i, best ? card.get(best) ?? best : "Other");
|
|
251
|
+
if (++v % 10 === 0 || v === todo.length)
|
|
252
|
+
onProgress({ phase: "looking", done: v, total: todo.length, etaMs: Math.round(((Date.now() - t2) / v) * (todo.length - v)) });
|
|
253
|
+
});
|
|
254
|
+
// the bucket is the fine label when enough pictures share it (or the creator asked for it), else its broader group; the labels stay as tags
|
|
255
|
+
// keep every picture's embedding next to the plan (float32, one row per item): labels added later are a dot product, not a rescan
|
|
256
|
+
if (o.embFile) {
|
|
257
|
+
const d = found.find((f) => f?.emb)?.emb?.length ?? 512;
|
|
258
|
+
const buf = new Float32Array(items.length * d);
|
|
259
|
+
found.forEach((f, i) => { if (f?.emb)
|
|
260
|
+
buf.set(f.emb, i * d); });
|
|
261
|
+
try {
|
|
262
|
+
fs_1.default.writeFileSync(o.embFile, Buffer.from(buf.buffer));
|
|
263
|
+
}
|
|
264
|
+
catch { /* relabelling will say it needs a rescan */ }
|
|
265
|
+
for (const f of found)
|
|
266
|
+
if (f)
|
|
267
|
+
delete f.emb;
|
|
268
|
+
}
|
|
269
|
+
const bucket = (0, topics_1.bucketFor)(found, set);
|
|
270
|
+
for (const i of todo) {
|
|
271
|
+
items[i].topic = bucket[i] ?? "Other";
|
|
272
|
+
items[i].tags = found[i]?.tags ?? [];
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
else {
|
|
276
|
+
const analyze = vision ? await visionAnalyzer() : null;
|
|
277
|
+
if (analyze) {
|
|
278
|
+
let v = 0;
|
|
279
|
+
await pool(2, todo.filter((i) => items[i].kind === "image"), async (i) => {
|
|
280
|
+
try {
|
|
281
|
+
const a = await analyze(path_1.default.join(root, items[i].rel), "image");
|
|
282
|
+
items[i].subject = a.subject || "";
|
|
283
|
+
items[i].tags = a.keywords || [];
|
|
284
|
+
}
|
|
285
|
+
catch { /* leave untagged */ }
|
|
286
|
+
if (++v % 5 === 0 || v === todo.length)
|
|
287
|
+
onProgress({ phase: "looking", done: v, total: todo.length });
|
|
288
|
+
});
|
|
289
|
+
}
|
|
290
|
+
else
|
|
291
|
+
by = "folder"; // nothing here can tell what a picture shows
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
// with the picture model, labels already fold small subjects into their group; folding again would swallow a label the creator asked for
|
|
295
|
+
const buckets = (0, plan_1.planBuckets)(items, { by, minBucket: by === "topic" && items.some((it) => it.tags?.length) ? 1 : 3 });
|
|
296
|
+
if (thumbs) {
|
|
297
|
+
fs_1.default.mkdirSync(thumbs, { recursive: true });
|
|
298
|
+
let n = 0;
|
|
299
|
+
const t1 = Date.now();
|
|
300
|
+
await pool(4, items, async (it, i) => {
|
|
301
|
+
const dst = path_1.default.join(thumbs, `${i}.jpg`);
|
|
302
|
+
if (it.dupOf === undefined && !it.thumb && (await (0, images_1.makeThumb)(path_1.default.join(root, it.rel), it.kind, dst)))
|
|
303
|
+
it.thumb = dst;
|
|
304
|
+
if (it.dupOf !== undefined && it.thumb) {
|
|
305
|
+
try {
|
|
306
|
+
fs_1.default.rmSync(it.thumb, { force: true });
|
|
307
|
+
}
|
|
308
|
+
catch { /* ignore */ }
|
|
309
|
+
delete it.thumb;
|
|
310
|
+
}
|
|
311
|
+
if (++n % 10 === 0 || n === items.length)
|
|
312
|
+
onProgress({ phase: "thumbs", done: n, total: items.length, etaMs: Math.round(((Date.now() - t1) / n) * (items.length - n)) });
|
|
313
|
+
});
|
|
314
|
+
}
|
|
315
|
+
const id = crypto_1.default.randomBytes(4).toString("hex");
|
|
316
|
+
const plan = { id, root, createdAt: new Date().toISOString(), by, vision, labels: custom, totals: (0, plan_1.summarize)(items, buckets), items, buckets, moves: (0, plan_1.planMoves)(root, items, buckets) };
|
|
317
|
+
onProgress({ phase: "done" });
|
|
318
|
+
return plan;
|
|
319
|
+
}
|
|
320
|
+
/** Thumbnails for a saved plan's non-duplicate files (the opt-in My Story upload). Rewrites the plan file with each item's `thumb`. */
|
|
321
|
+
async function makeThumbsFor(planFile, dir) {
|
|
322
|
+
const plan = JSON.parse(fs_1.default.readFileSync(planFile, "utf8"));
|
|
323
|
+
fs_1.default.mkdirSync(dir, { recursive: true });
|
|
324
|
+
await pool(4, plan.items, async (it, i) => {
|
|
325
|
+
const dst = path_1.default.join(dir, `${i}.jpg`);
|
|
326
|
+
if (it.dupOf === undefined && (fs_1.default.existsSync(dst) || (await (0, images_1.makeThumb)(path_1.default.join(plan.root, it.rel), it.kind, dst))))
|
|
327
|
+
it.thumb = dst;
|
|
328
|
+
});
|
|
329
|
+
fs_1.default.writeFileSync(planFile, JSON.stringify(plan));
|
|
330
|
+
return plan;
|
|
331
|
+
}
|
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.CUSTOM_MIN = exports.OTHER = exports.VOCAB = void 0;
|
|
7
|
+
exports.labelSet = labelSet;
|
|
8
|
+
exports.pickLabels = pickLabels;
|
|
9
|
+
exports.bucketFor = bucketFor;
|
|
10
|
+
exports.labelsFromEmbedding = labelsFromEmbedding;
|
|
11
|
+
exports.textEmbeddings = textEmbeddings;
|
|
12
|
+
exports.labeller = labeller;
|
|
13
|
+
/**
|
|
14
|
+
* Labels ON THIS COMPUTER: CLIP (a small open image model, ~150 MB, downloaded once and cached) compares each picture with a vocabulary of everyday
|
|
15
|
+
* subjects plus any labels the creator typed ("tennis", "kids at the beach"). Nothing leaves the computer.
|
|
16
|
+
*
|
|
17
|
+
* Every picture gets up to three labels (its tags, for search and filters) and one bucket. The bucket is the fine label ("Tennis") when enough
|
|
18
|
+
* pictures share it, or when the creator asked for it; otherwise the broader group it belongs to ("Sport & wellness"), the reel's cards. So labels
|
|
19
|
+
* appear as the folder needs them rather than from a fixed list, and a one-off subject never gets a card of its own.
|
|
20
|
+
*
|
|
21
|
+
* The text side is embedded once per scan; each picture is one vision pass and a dot product, so 60 labels cost the same as 14.
|
|
22
|
+
* Optional: when @huggingface/transformers (or sharp, which makes its input) is missing, labeller() returns null and the scan sorts by folder.
|
|
23
|
+
*/
|
|
24
|
+
const fs_1 = __importDefault(require("fs"));
|
|
25
|
+
const os_1 = __importDefault(require("os"));
|
|
26
|
+
const path_1 = __importDefault(require("path"));
|
|
27
|
+
/** [label, what CLIP is asked, the broader group it falls back to]. The groups are the reel's cards. */
|
|
28
|
+
exports.VOCAB = [
|
|
29
|
+
["Selfies", "a selfie of a person looking at the camera", "Selfies & talking"],
|
|
30
|
+
["Mirror selfies", "a mirror selfie", "Selfies & talking"],
|
|
31
|
+
["Talking to camera", "a person talking to the camera", "Selfies & talking"],
|
|
32
|
+
["Group photos", "a group of friends posing together", "Events"],
|
|
33
|
+
["Outfits", "a person showing an outfit or clothes", "Outfits"],
|
|
34
|
+
["Shoes & accessories", "shoes, a bag or jewellery", "Outfits"],
|
|
35
|
+
["Swimwear", "a person in a swimsuit or bikini", "Outfits"],
|
|
36
|
+
["Rooftops & terraces", "a rooftop terrace with a city view", "Travel & vibe"],
|
|
37
|
+
["Food", "a plate of food", "Food & cafés"],
|
|
38
|
+
["Coffee & drinks", "a coffee or a drink", "Food & cafés"],
|
|
39
|
+
["Desserts", "a cake or a dessert", "Food & cafés"],
|
|
40
|
+
["Restaurants", "people eating at a restaurant", "Food & cafés"],
|
|
41
|
+
["City streets", "a city street", "Travel & vibe"],
|
|
42
|
+
["Landmarks", "a famous landmark or monument", "Travel & vibe"],
|
|
43
|
+
["Airports & flights", "an airport or the view from an airplane", "Travel & vibe"],
|
|
44
|
+
["Hotels", "a hotel room or a resort", "Travel & vibe"],
|
|
45
|
+
["Night lights", "a city at night", "Travel & vibe"],
|
|
46
|
+
["Beach", "a beach", "Nature"],
|
|
47
|
+
["Mountains", "mountains or hiking", "Nature"],
|
|
48
|
+
["Sunsets", "a sunset or sunrise", "Nature"],
|
|
49
|
+
["Snow", "snow and winter", "Nature"],
|
|
50
|
+
["Flowers & gardens", "flowers or a garden", "Nature"],
|
|
51
|
+
["Parks", "a park with trees and grass", "Nature"],
|
|
52
|
+
["Water & boats", "a lake, a river or a boat", "Nature"],
|
|
53
|
+
["Home interiors", "a living room or a bedroom", "Aesthetic spaces"],
|
|
54
|
+
["Architecture", "a building or architecture", "Aesthetic spaces"],
|
|
55
|
+
["Tennis", "a person playing tennis or a tennis court", "Sport & wellness"],
|
|
56
|
+
["Golf", "golf", "Sport & wellness"],
|
|
57
|
+
["Running", "a person running", "Sport & wellness"],
|
|
58
|
+
["Gym", "a gym or lifting weights", "Sport & wellness"],
|
|
59
|
+
["Yoga", "yoga or stretching", "Sport & wellness"],
|
|
60
|
+
["Swimming", "a swimming pool or swimming", "Sport & wellness"],
|
|
61
|
+
["Basketball", "basketball", "Sport & wellness"],
|
|
62
|
+
["Soccer", "soccer or football", "Sport & wellness"],
|
|
63
|
+
["Cycling", "a bicycle or cycling", "Sport & wellness"],
|
|
64
|
+
["Skiing", "skiing or snowboarding", "Sport & wellness"],
|
|
65
|
+
["Concerts", "a concert or live music", "Music & hobby"],
|
|
66
|
+
["Instruments", "a musical instrument", "Music & hobby"],
|
|
67
|
+
["Art & crafts", "a painting, drawing or crafts", "Music & hobby"],
|
|
68
|
+
["Games", "video games or board games", "Music & hobby"],
|
|
69
|
+
["Desk & laptop", "a laptop on a desk", "Work from home"],
|
|
70
|
+
["Meetings & talks", "a meeting, a presentation or someone on stage", "Work from home"],
|
|
71
|
+
["Whiteboards & slides", "a whiteboard or a presentation slide", "Text & captions"],
|
|
72
|
+
["Screenshots", "a screenshot of a phone or computer screen", "Text & captions"],
|
|
73
|
+
["Documents & receipts", "a document, a receipt or a printed page", "Text & captions"],
|
|
74
|
+
["Charts", "a chart or a graph", "Text & captions"],
|
|
75
|
+
["Dogs", "a dog", "Pets"],
|
|
76
|
+
["Cats", "a cat", "Pets"],
|
|
77
|
+
["Babies", "a baby", "Family"],
|
|
78
|
+
["Kids", "children playing", "Family"],
|
|
79
|
+
["Family", "a family together", "Family"],
|
|
80
|
+
["Weddings", "a wedding", "Events"],
|
|
81
|
+
["Birthdays", "a birthday party or a cake with candles", "Events"],
|
|
82
|
+
["Parties", "a party with people", "Events"],
|
|
83
|
+
["Holidays", "a christmas tree or holiday decorations", "Events"],
|
|
84
|
+
["Cars", "a car", "Products"],
|
|
85
|
+
["Products", "a product photo of an object", "Products"],
|
|
86
|
+
["Gadgets", "a phone, a camera or a gadget", "Products"],
|
|
87
|
+
["Shopping", "a shop or shopping", "Products"],
|
|
88
|
+
];
|
|
89
|
+
exports.OTHER = "Other";
|
|
90
|
+
const MODEL = "Xenova/clip-vit-base-patch32";
|
|
91
|
+
/** a label counts for a picture when it is at least this share of the softmax and within this ratio of the best one; below BEST_MIN the picture is "Other" */
|
|
92
|
+
const TAG_MIN = 0.08, TAG_RATIO = 0.35, BEST_MIN = 0.12;
|
|
93
|
+
/**
|
|
94
|
+
* The cosine a picture needs with a creator's own label (CLIP ViT-B/32, "a photo of …"). Measured on Allen's 3,911 pictures for "tennis": the top
|
|
95
|
+
* 40 (cos >= 0.286) were all tennis, ranks 60-100 (0.267-0.248) rooftop and pool shots; 0.27 keeps the courts and drops the rest.
|
|
96
|
+
*/
|
|
97
|
+
exports.CUSTOM_MIN = 0.27;
|
|
98
|
+
/** The vocabulary plus the creator's own labels (theirs win when the name matches; each is its own card, never folded into a group). */
|
|
99
|
+
function labelSet(custom = []) {
|
|
100
|
+
const mine = [...new Set(custom.map((s) => String(s).trim().slice(0, 40)).filter(Boolean))];
|
|
101
|
+
const lower = new Set(mine.map((s) => s.toLowerCase()));
|
|
102
|
+
const base = exports.VOCAB.filter(([n]) => !lower.has(n.toLowerCase()));
|
|
103
|
+
const titled = mine.map((s) => s[0].toUpperCase() + s.slice(1));
|
|
104
|
+
// "tennis" typed by the creator keeps the built-in, descriptive prompt ("a person playing tennis or a tennis court"): the bare word matched
|
|
105
|
+
// rooftops, pools and sporty outfits on Allen's folder
|
|
106
|
+
const known = new Map(exports.VOCAB.map(([n, p]) => [n.toLowerCase(), p]));
|
|
107
|
+
return {
|
|
108
|
+
names: [...titled, ...base.map(([n]) => n)],
|
|
109
|
+
prompts: [...mine.map((m) => known.get(m.toLowerCase()) ?? m), ...base.map(([, p]) => p)],
|
|
110
|
+
groups: [...titled, ...base.map(([, , g]) => g)],
|
|
111
|
+
custom: new Set(titled),
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
/** From one picture's softmax over the label set: its tags (best first) and its best label, or none when nothing is clear. */
|
|
115
|
+
function pickLabels(probs, set) {
|
|
116
|
+
const order = probs.map((p, i) => [p, i]).sort((a, b) => b[0] - a[0]);
|
|
117
|
+
const top = order[0]?.[0] ?? 0;
|
|
118
|
+
const tags = order.filter(([p], k) => k < 3 && p >= TAG_MIN && p >= top * TAG_RATIO).map(([, i]) => set.names[i]);
|
|
119
|
+
return { tags, best: top >= BEST_MIN ? set.names[order[0][1]] : null };
|
|
120
|
+
}
|
|
121
|
+
/**
|
|
122
|
+
* Buckets from the labels: a fine label gets its own card when the creator asked for it or when enough pictures share it
|
|
123
|
+
* (at least `min`, default max(6, 2% of the pictures)); otherwise its pictures go to the label's group card.
|
|
124
|
+
*/
|
|
125
|
+
function bucketFor(labels, set, min) {
|
|
126
|
+
const n = labels.filter(Boolean).length;
|
|
127
|
+
const need = min ?? Math.max(6, Math.ceil(n * 0.02));
|
|
128
|
+
const counts = new Map();
|
|
129
|
+
for (const l of labels)
|
|
130
|
+
if (l?.best)
|
|
131
|
+
counts.set(l.best, (counts.get(l.best) ?? 0) + 1);
|
|
132
|
+
const group = new Map(set.names.map((nm, i) => [nm, set.groups[i]]));
|
|
133
|
+
return labels.map((l) => {
|
|
134
|
+
if (!l)
|
|
135
|
+
return undefined;
|
|
136
|
+
if (!l.best)
|
|
137
|
+
return exports.OTHER;
|
|
138
|
+
if (set.custom.has(l.best) || (counts.get(l.best) ?? 0) >= need)
|
|
139
|
+
return l.best;
|
|
140
|
+
return group.get(l.best) ?? exports.OTHER;
|
|
141
|
+
});
|
|
142
|
+
}
|
|
143
|
+
let model;
|
|
144
|
+
async function loadModel() {
|
|
145
|
+
// transformers is ESM-only; tsc (CommonJS) would turn a literal import() into require()
|
|
146
|
+
const esm = new Function("m", "return import(m)");
|
|
147
|
+
const tf = await esm("@huggingface/transformers");
|
|
148
|
+
tf.env.cacheDir = path_1.default.join(os_1.default.homedir(), ".viralspin", "models");
|
|
149
|
+
fs_1.default.mkdirSync(tf.env.cacheDir, { recursive: true });
|
|
150
|
+
const [tokenizer, text, processor, vision] = await Promise.all([
|
|
151
|
+
tf.AutoTokenizer.from_pretrained(MODEL),
|
|
152
|
+
tf.CLIPTextModelWithProjection.from_pretrained(MODEL),
|
|
153
|
+
tf.AutoProcessor.from_pretrained(MODEL),
|
|
154
|
+
tf.CLIPVisionModelWithProjection.from_pretrained(MODEL),
|
|
155
|
+
]);
|
|
156
|
+
return { tf, tokenizer, text, processor, vision };
|
|
157
|
+
}
|
|
158
|
+
const unit = (v, off, d) => { let s = 0; for (let i = 0; i < d; i++)
|
|
159
|
+
s += v[off + i] ** 2; const k = 1 / Math.sqrt(s || 1); return Array.from({ length: d }, (_, i) => v[off + i] * k); };
|
|
160
|
+
/** Labels from a stored picture embedding (unit length) against the label set's text embeddings: the same answer as the scan gave, without the picture. */
|
|
161
|
+
function labelsFromEmbedding(v, T, set) {
|
|
162
|
+
const cos = T.map((t) => { let d = 0; for (let i = 0; i < t.length; i++)
|
|
163
|
+
d += t[i] * v[i]; return d; });
|
|
164
|
+
const logits = cos.map((c) => 100 * c);
|
|
165
|
+
const mx = Math.max(...logits);
|
|
166
|
+
const e = logits.map((x) => Math.exp(x - mx));
|
|
167
|
+
const z = e.reduce((x, y) => x + y, 0);
|
|
168
|
+
// a label the creator asked for must truly match, not merely beat the other labels: below CUSTOM_MIN it is not offered for this picture
|
|
169
|
+
const probs = e.map((x, i) => (set.custom.has(set.names[i]) && cos[i] < exports.CUSTOM_MIN ? 0 : x / z));
|
|
170
|
+
return pickLabels(probs, set);
|
|
171
|
+
}
|
|
172
|
+
/** The label set's text side, embedded once (unit vectors), or null when the model can not run here. */
|
|
173
|
+
async function textEmbeddings(set) {
|
|
174
|
+
if (!model)
|
|
175
|
+
model = loadModel().catch(() => null);
|
|
176
|
+
const m = await model;
|
|
177
|
+
if (!m)
|
|
178
|
+
return null;
|
|
179
|
+
try {
|
|
180
|
+
const { text_embeds } = await m.text(m.tokenizer(set.prompts.map((p) => `a photo of ${p}`), { padding: true, truncation: true }));
|
|
181
|
+
const d = text_embeds.dims[1];
|
|
182
|
+
return set.prompts.map((_, i) => unit(text_embeds.data, i * d, d));
|
|
183
|
+
}
|
|
184
|
+
catch {
|
|
185
|
+
return null;
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
/**
|
|
189
|
+
* A labeller for this label set, or null when this install cannot run the model. The model loads once per process (the first run downloads it).
|
|
190
|
+
* Each answer carries the picture's embedding too: the scan keeps it, so a label added later ("tennis") re-sorts the folder in seconds.
|
|
191
|
+
*/
|
|
192
|
+
async function labeller(set) {
|
|
193
|
+
const T = await textEmbeddings(set);
|
|
194
|
+
const m = await model;
|
|
195
|
+
if (!T || !m)
|
|
196
|
+
return null;
|
|
197
|
+
const d = T[0]?.length ?? 512;
|
|
198
|
+
return async (rgb, w, h) => {
|
|
199
|
+
const img = new m.tf.RawImage(new Uint8ClampedArray(rgb), w, h, 3);
|
|
200
|
+
const { image_embeds } = await m.vision(await m.processor(img));
|
|
201
|
+
const v = unit(image_embeds.data, 0, d);
|
|
202
|
+
return { ...labelsFromEmbedding(v, T, set), emb: v };
|
|
203
|
+
};
|
|
204
|
+
}
|