@mevdragon/vidfarm-devcli 0.18.0 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/editor-capabilities/SKILL.md +166 -0
- package/.agents/skills/editor-capabilities/references/re-theme-walkthrough.md +58 -0
- package/.agents/skills/vidfarm-media/SKILL.md +43 -4
- package/SKILL.director.md +190 -18
- package/SKILL.platform.md +8 -4
- package/demo/dist/app.css +1 -1
- package/demo/dist/app.js +77 -75
- package/demo/dist/favicon.ico +0 -0
- package/dist/src/app.js +3031 -300
- package/dist/src/cli.js +1549 -69
- package/dist/src/config.js +7 -0
- package/dist/src/devcli/clip-store.js +29 -2
- package/dist/src/devcli/clips.js +55 -9
- package/dist/src/devcli/composition-edit.js +658 -8
- package/dist/src/devcli/local-backend.js +123 -0
- package/dist/src/devcli/migrate-local.js +140 -0
- package/dist/src/devcli/sync.js +311 -0
- package/dist/src/devcli/timeline-edit.js +490 -0
- package/dist/src/editor-chat.js +56 -17
- package/dist/src/frontend/discover-client.js +130 -0
- package/dist/src/frontend/discover-store.js +23 -0
- package/dist/src/frontend/file-directory.js +995 -0
- package/dist/src/frontend/homepage-view.js +3 -3
- package/dist/src/frontend/template-editor-chat.js +28 -22
- package/dist/src/landing-page.js +24 -7
- package/dist/src/page-shell.js +26 -2
- package/dist/src/primitive-registry.js +409 -4
- package/dist/src/reskin/agency-page.js +1 -1
- package/dist/src/reskin/calendar-page.js +2 -1
- package/dist/src/reskin/chat-page.js +420 -85
- package/dist/src/reskin/discover-page.js +731 -39
- package/dist/src/reskin/document.js +1311 -387
- package/dist/src/reskin/help-page.js +1 -0
- package/dist/src/reskin/inpaint-clipper-page.js +649 -0
- package/dist/src/reskin/inpaint-page.js +2459 -446
- package/dist/src/reskin/inpaint-video-page.js +1339 -0
- package/dist/src/reskin/library-page.js +1168 -228
- package/dist/src/reskin/login-page.js +6 -6
- package/dist/src/reskin/pricing-page.js +2 -0
- package/dist/src/reskin/settings-page.js +55 -10
- package/dist/src/reskin/theme.js +365 -20
- package/dist/src/services/billing.js +4 -0
- package/dist/src/services/clip-curation/gemini.js +5 -0
- package/dist/src/services/clip-curation/hunt.js +81 -1
- package/dist/src/services/clip-curation/index.js +2 -1
- package/dist/src/services/clip-curation/local-agent.js +4 -3
- package/dist/src/services/clip-curation/media-select.js +85 -0
- package/dist/src/services/clip-curation/query.js +5 -1
- package/dist/src/services/clip-curation/refine.js +50 -20
- package/dist/src/services/clip-curation/scan.js +10 -3
- package/dist/src/services/clip-curation/taxonomy.js +3 -1
- package/dist/src/services/clip-curation/taxonomy.v1.json +13 -1
- package/dist/src/services/clip-records.js +42 -1
- package/dist/src/services/clip-search.js +43 -13
- package/dist/src/services/file-directory.js +117 -0
- package/dist/src/services/hyperframes.js +283 -3
- package/dist/src/services/serverless-records.js +43 -0
- package/dist/src/services/storage.js +47 -2
- package/dist/src/services/upstream.js +5 -5
- package/dist/src/template-editor-shell.js +16 -2
- package/package.json +1 -1
- package/public/assets/discover-client-app.js +1 -0
- package/public/assets/file-directory-app.js +2 -0
- package/public/assets/homepage-client-app.js +12 -12
- package/public/assets/page-runtime-client-app.js +24 -24
- package/public/assets/placeholders/scene-placeholder.png +0 -0
- package/src/assets/favicon.ico +0 -0
- package/src/assets/logo-vidfarm.png +0 -0
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
// In-process LOCAL backend for the devcli — the "local" half of local-first,
|
|
2
|
+
// dual local/cloud operation.
|
|
3
|
+
//
|
|
4
|
+
// The whole Vidfarm backend already runs over disk-backed drivers when
|
|
5
|
+
// RECORDS_DRIVER=local + STORAGE_DRIVER=local (the same drivers `vidfarm serve`
|
|
6
|
+
// uses). Rather than re-implement the directory / files / raws logic a second
|
|
7
|
+
// time, we boot the real Hono `app` in-process and drive it with `app.request()`
|
|
8
|
+
// — Hono's fetch interface runs the full middleware + handler stack WITHOUT
|
|
9
|
+
// binding a socket. So `vidfarm directory ls --local` and `vidfarm serve` share
|
|
10
|
+
// one on-disk backend, and a `clips scan --local` is instantly browsable in a
|
|
11
|
+
// running serve UI.
|
|
12
|
+
//
|
|
13
|
+
// IMPORTANT (env-before-import): `config.ts` binds the driver + data dir from
|
|
14
|
+
// env at module load. We MUST set the local env BEFORE the first `import` of
|
|
15
|
+
// `../app.js` / any service. `cli.ts`'s static import graph deliberately does
|
|
16
|
+
// not reach `config.js`, so setting env here (then dynamic-importing) flips the
|
|
17
|
+
// drivers correctly — exactly the discipline `runServeCommand` follows.
|
|
18
|
+
import path from "node:path";
|
|
19
|
+
import { homedir } from "node:os";
|
|
20
|
+
/** The durable per-user local home. Everything the local backend persists lives
|
|
21
|
+
* under here: records + storage at `<home>/data`, legacy clips at
|
|
22
|
+
* `<home>/clips.db`, provider config at `<home>/config.json`. */
|
|
23
|
+
export function resolveLocalHome(overrideDir) {
|
|
24
|
+
if (overrideDir)
|
|
25
|
+
return path.resolve(overrideDir);
|
|
26
|
+
if (process.env.VIDFARM_HOME)
|
|
27
|
+
return path.resolve(process.env.VIDFARM_HOME);
|
|
28
|
+
return path.join(homedir(), ".vidfarm");
|
|
29
|
+
}
|
|
30
|
+
/** The data dir (records + storage) for the local backend. */
|
|
31
|
+
export function resolveLocalDataDir(overrideDir) {
|
|
32
|
+
return path.join(resolveLocalHome(overrideDir), "data");
|
|
33
|
+
}
|
|
34
|
+
let cached = null;
|
|
35
|
+
/**
|
|
36
|
+
* Boot (once) the in-process local backend and return a handle. Idempotent:
|
|
37
|
+
* repeated calls return the same booted app. Safe to call from any local
|
|
38
|
+
* command; cloud-only commands never call this, so the cloud path is untouched.
|
|
39
|
+
*/
|
|
40
|
+
export async function withLocalBackend(opts = {}) {
|
|
41
|
+
if (cached)
|
|
42
|
+
return cached;
|
|
43
|
+
const home = resolveLocalHome(opts.home);
|
|
44
|
+
const dataDir = resolveLocalDataDir(opts.home);
|
|
45
|
+
// Force the fully-local drivers, and blank every knob that would otherwise
|
|
46
|
+
// route this in-process box at the cloud (mirrors runServeCommand). These
|
|
47
|
+
// MUST be set before app.js / config.js is imported below.
|
|
48
|
+
process.env.RECORDS_DRIVER = "local";
|
|
49
|
+
process.env.STORAGE_DRIVER = "local";
|
|
50
|
+
process.env.AWS_S3_BUCKET = "";
|
|
51
|
+
process.env.VIDFARM_JOB_STATE_MACHINE_ARN = "";
|
|
52
|
+
process.env.VIDFARM_CLIP_SCAN_STATE_MACHINE_ARN = "";
|
|
53
|
+
process.env.VIDFARM_PRIMITIVE_MEDIA_LAMBDA_URL = "";
|
|
54
|
+
process.env.VIDFARM_PRIMITIVE_MEDIA_LAMBDA_SECRET = "";
|
|
55
|
+
// The one records-touching service that bypasses the local shim and talks to a
|
|
56
|
+
// real DynamoDB table via middleware — blank it or every /api/* 500s when a
|
|
57
|
+
// dev-shell .env carries a table name. In-process service calls don't hit
|
|
58
|
+
// middleware, but app.request() does, so this must be blank.
|
|
59
|
+
process.env.VIDFARM_RATE_LIMITS_TABLE_NAME = "";
|
|
60
|
+
process.env.VIDFARM_DATA_DIR = dataDir;
|
|
61
|
+
if (!process.env.NODE_ENV)
|
|
62
|
+
process.env.NODE_ENV = "development";
|
|
63
|
+
process.env.HYPERFRAMES_NO_TELEMETRY ||= "1";
|
|
64
|
+
process.env.HYPERFRAMES_SKIP_SKILLS ||= "1";
|
|
65
|
+
// No cloud passthrough by default for pure-local commands; sync/`--cloud` reach
|
|
66
|
+
// the cloud host directly via the devcli's own apiRequest, not this app.
|
|
67
|
+
process.env.VIDFARM_UPSTREAM_HOST ||= "";
|
|
68
|
+
process.env.VIDFARM_UPSTREAM_API_KEY ||= "";
|
|
69
|
+
// A stable local bootstrap key. The same value auto-provisions the local
|
|
70
|
+
// customer (tryBootstrapFromEnv) and authenticates app.request() calls. An
|
|
71
|
+
// empty/whitespace override falls through to the env key or the fixed default
|
|
72
|
+
// (so `--api-key ""` never breaks the local backend).
|
|
73
|
+
const apiKey = (opts.apiKey && opts.apiKey.trim()) ||
|
|
74
|
+
(process.env.VIDFARM_API_KEY && process.env.VIDFARM_API_KEY.trim()) ||
|
|
75
|
+
"vidfarm-local-dev-key";
|
|
76
|
+
process.env.VIDFARM_API_KEY = apiKey;
|
|
77
|
+
// Dynamic imports AFTER env is set, so config.ts binds the local drivers.
|
|
78
|
+
const { ServerlessAuthService } = await import("../services/serverless-auth.js");
|
|
79
|
+
const customer = await new ServerlessAuthService().authenticate(apiKey);
|
|
80
|
+
const appMod = await import("../app.js");
|
|
81
|
+
const app = appMod.default;
|
|
82
|
+
cached = { app, apiKey, ownerId: customer.id, home };
|
|
83
|
+
return cached;
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* Dispatch a REST call to the in-process local app. Mirrors the cloud
|
|
87
|
+
* `apiRequest` shape so commands can branch on target without reshaping results.
|
|
88
|
+
*/
|
|
89
|
+
export async function localApiRequest(input) {
|
|
90
|
+
const backend = await withLocalBackend({ home: input.home, apiKey: input.auth?.apiKey });
|
|
91
|
+
// A synthetic origin — only the path + query matter to Hono routing.
|
|
92
|
+
const url = new URL(input.path, "http://vidfarm.local");
|
|
93
|
+
for (const [key, value] of Object.entries(input.query ?? {})) {
|
|
94
|
+
if (value !== undefined && value !== null && value !== "")
|
|
95
|
+
url.searchParams.set(key, value);
|
|
96
|
+
}
|
|
97
|
+
const headers = {
|
|
98
|
+
accept: "application/json",
|
|
99
|
+
"vidfarm-api-key": input.auth?.apiKey || backend.apiKey
|
|
100
|
+
};
|
|
101
|
+
if (input.auth?.shareToken)
|
|
102
|
+
headers["vidfarm-share-token"] = input.auth.shareToken;
|
|
103
|
+
let body;
|
|
104
|
+
if (input.body !== undefined) {
|
|
105
|
+
headers["content-type"] = "application/json";
|
|
106
|
+
body = JSON.stringify(input.body);
|
|
107
|
+
}
|
|
108
|
+
const res = await backend.app.request(url.pathname + url.search, {
|
|
109
|
+
method: input.method.toUpperCase(),
|
|
110
|
+
headers,
|
|
111
|
+
body
|
|
112
|
+
});
|
|
113
|
+
const text = await res.text();
|
|
114
|
+
let json = null;
|
|
115
|
+
try {
|
|
116
|
+
json = text ? JSON.parse(text) : null;
|
|
117
|
+
}
|
|
118
|
+
catch {
|
|
119
|
+
json = null;
|
|
120
|
+
}
|
|
121
|
+
return { status: res.status, ok: res.ok, json, text };
|
|
122
|
+
}
|
|
123
|
+
//# sourceMappingURL=local-backend.js.map
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
// `vidfarm migrate-local` — bridge the legacy standalone SQLite clip library
|
|
2
|
+
// (~/.vidfarm/clips.db, sqlite-vec) into the UNIFIED local backend that
|
|
3
|
+
// `vidfarm serve` + the file directory read (disk-backed clipRecords + storage
|
|
4
|
+
// under ~/.vidfarm/data). After migrating, every locally-hunted clip is
|
|
5
|
+
// browsable at /raws and semantically searchable with the SAME 768-dim vectors
|
|
6
|
+
// (identical model + text builder), so nothing is re-embedded.
|
|
7
|
+
//
|
|
8
|
+
// Idempotent: putClip is an upsert keyed by clip_id, and a `migrated.json` marker
|
|
9
|
+
// stops the lazy auto-offer from nagging. The SQLite db is left untouched.
|
|
10
|
+
import path from "node:path";
|
|
11
|
+
import { existsSync, writeFileSync } from "node:fs";
|
|
12
|
+
import { parseArgs } from "node:util";
|
|
13
|
+
const GREEN = "\x1b[32m";
|
|
14
|
+
const DIM = "\x1b[2m";
|
|
15
|
+
const BOLD = "\x1b[1m";
|
|
16
|
+
const RESET = "\x1b[0m";
|
|
17
|
+
/** Slugify a source filename into the folder clips group under (mirrors the
|
|
18
|
+
* serve-box source-slug so migrated clips land in the same /raws/<slug> folder). */
|
|
19
|
+
function sourceSlug(filename) {
|
|
20
|
+
return String(filename || "")
|
|
21
|
+
.toLowerCase()
|
|
22
|
+
.replace(/\.[a-z0-9]+$/i, "")
|
|
23
|
+
.replace(/[^a-z0-9]+/g, "-")
|
|
24
|
+
.replace(/^-+|-+$/g, "")
|
|
25
|
+
.slice(0, 60);
|
|
26
|
+
}
|
|
27
|
+
export async function runMigrateLocalCommand(argv) {
|
|
28
|
+
const { values } = parseArgs({
|
|
29
|
+
args: argv,
|
|
30
|
+
allowPositionals: true,
|
|
31
|
+
options: {
|
|
32
|
+
home: { type: "string" },
|
|
33
|
+
"dry-run": { type: "boolean", default: false },
|
|
34
|
+
force: { type: "boolean", default: false }
|
|
35
|
+
}
|
|
36
|
+
});
|
|
37
|
+
const { withLocalBackend, resolveLocalHome, resolveLocalDataDir } = await import("./local-backend.js");
|
|
38
|
+
const home = resolveLocalHome(values.home);
|
|
39
|
+
const dbPath = path.join(home, "clips.db");
|
|
40
|
+
if (!existsSync(dbPath)) {
|
|
41
|
+
console.log(`No local SQLite clip library at ${DIM}${dbPath}${RESET} — nothing to migrate.`);
|
|
42
|
+
return;
|
|
43
|
+
}
|
|
44
|
+
const markerPath = path.join(resolveLocalDataDir(values.home), "migrated.json");
|
|
45
|
+
if (existsSync(markerPath) && !values.force) {
|
|
46
|
+
console.log(`Already migrated (${DIM}${markerPath}${RESET}). Re-run with ${BOLD}--force${RESET} to migrate again.`);
|
|
47
|
+
return;
|
|
48
|
+
}
|
|
49
|
+
// Read the legacy SQLite store (read-only) — the ClipStore class is retained
|
|
50
|
+
// solely for this bridge.
|
|
51
|
+
const { ClipStore } = await import("./clip-store.js");
|
|
52
|
+
const store = new ClipStore(home);
|
|
53
|
+
const clips = store.listClips();
|
|
54
|
+
store.attachEmbeddings(clips);
|
|
55
|
+
const sources = store.listSourceVideos();
|
|
56
|
+
const presets = store.listPresets().filter((p) => !p.builtin);
|
|
57
|
+
const libModel = store.getLibraryEmbeddingModel();
|
|
58
|
+
if (clips.length === 0 && sources.length === 0) {
|
|
59
|
+
store.close();
|
|
60
|
+
console.log("The local SQLite library is empty — nothing to migrate.");
|
|
61
|
+
return;
|
|
62
|
+
}
|
|
63
|
+
if (values["dry-run"]) {
|
|
64
|
+
store.close();
|
|
65
|
+
console.log(`${BOLD}migrate-local (dry run)${RESET} ${DIM}${home}${RESET}`);
|
|
66
|
+
console.log(` ${clips.length} clip${clips.length === 1 ? "" : "s"} · ${sources.length} source${sources.length === 1 ? "" : "s"} · ${presets.length} custom preset${presets.length === 1 ? "" : "s"}`);
|
|
67
|
+
console.log(` embedding model: ${libModel ?? DIM + "(none — clips keyword-only)" + RESET}`);
|
|
68
|
+
console.log(` → would write into the unified backend at ${DIM}${resolveLocalDataDir(values.home)}${RESET}`);
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
// Boot the local backend (sets drivers + resolves the stable owner id), then
|
|
72
|
+
// reach the disk-backed services directly.
|
|
73
|
+
const backend = await withLocalBackend({ home: values.home });
|
|
74
|
+
const owner = backend.ownerId;
|
|
75
|
+
const { clipRecords } = await import("../services/clip-records.js");
|
|
76
|
+
const { storage } = await import("../services/storage.js");
|
|
77
|
+
const now = new Date().toISOString();
|
|
78
|
+
// Sources first (so /raws folder grouping + source lookups resolve).
|
|
79
|
+
let sourceCount = 0;
|
|
80
|
+
for (const s of sources) {
|
|
81
|
+
await clipRecords.putSource({
|
|
82
|
+
source_video_id: s.source_video_id,
|
|
83
|
+
owner_id: owner,
|
|
84
|
+
filename: s.filename,
|
|
85
|
+
raw_s3_uri: s.source_path ?? undefined,
|
|
86
|
+
duration_sec: s.duration_sec ?? undefined,
|
|
87
|
+
clip_count: s.clip_count ?? 0,
|
|
88
|
+
tier: s.tier ?? undefined,
|
|
89
|
+
status: "complete",
|
|
90
|
+
created_at: s.scanned_at ?? now,
|
|
91
|
+
updated_at: s.scanned_at ?? now
|
|
92
|
+
});
|
|
93
|
+
sourceCount += 1;
|
|
94
|
+
}
|
|
95
|
+
// Clips: copy the mp4/thumb bytes into the storage driver at the canonical
|
|
96
|
+
// unified keys, then upsert the record carrying its embedding verbatim.
|
|
97
|
+
let clipCount = 0;
|
|
98
|
+
let withVideo = 0;
|
|
99
|
+
for (const clip of clips) {
|
|
100
|
+
const clipId = clip.clip_id;
|
|
101
|
+
const srcMp4 = path.join(store.paths.clipsDir, `${clipId}.mp4`);
|
|
102
|
+
const clipKey = `clips/${owner}/${clipId}.mp4`;
|
|
103
|
+
if (existsSync(srcMp4)) {
|
|
104
|
+
await storage.putFile(clipKey, srcMp4, "video/mp4");
|
|
105
|
+
withVideo += 1;
|
|
106
|
+
}
|
|
107
|
+
const srcThumb = path.join(store.paths.thumbsDir, `${clipId}.jpg`);
|
|
108
|
+
let thumbKey = "";
|
|
109
|
+
if (existsSync(srcThumb)) {
|
|
110
|
+
thumbKey = `thumbs/${owner}/${clipId}.jpg`;
|
|
111
|
+
await storage.putFile(thumbKey, srcThumb, "image/jpeg");
|
|
112
|
+
}
|
|
113
|
+
const unified = {
|
|
114
|
+
...clip,
|
|
115
|
+
owner_id: owner,
|
|
116
|
+
file_path: clipKey,
|
|
117
|
+
thumbnail_path: thumbKey,
|
|
118
|
+
folder_path: clip.folder_path && clip.folder_path.trim() ? clip.folder_path : sourceSlug(clip.source_filename),
|
|
119
|
+
// Vectors are the same 768-dim space; stamp the library model so cloud/
|
|
120
|
+
// local search stays cosine-comparable.
|
|
121
|
+
...(clip.embedding && clip.embedding.length && !clip.embedding_model && libModel
|
|
122
|
+
? { embedding_model: libModel }
|
|
123
|
+
: {})
|
|
124
|
+
};
|
|
125
|
+
await clipRecords.putClip(unified);
|
|
126
|
+
clipCount += 1;
|
|
127
|
+
}
|
|
128
|
+
// Custom presets (built-ins are seeded on both sides already).
|
|
129
|
+
let presetCount = 0;
|
|
130
|
+
for (const p of presets) {
|
|
131
|
+
await clipRecords.putPreset(owner, p);
|
|
132
|
+
presetCount += 1;
|
|
133
|
+
}
|
|
134
|
+
store.close();
|
|
135
|
+
writeFileSync(markerPath, JSON.stringify({ migrated_at: now, clips: clipCount, sources: sourceCount, presets: presetCount, embedding_model: libModel }, null, 2));
|
|
136
|
+
console.log(`${GREEN}✓ Migrated the local clip library into the unified backend.${RESET}`);
|
|
137
|
+
console.log(` ${clipCount} clip${clipCount === 1 ? "" : "s"} (${withVideo} with video) · ${sourceCount} source${sourceCount === 1 ? "" : "s"} · ${presetCount} preset${presetCount === 1 ? "" : "s"}`);
|
|
138
|
+
console.log(` ${DIM}Browse them:${RESET} vidfarm directory ls /raws ${DIM}or${RESET} vidfarm serve → /library/raws`);
|
|
139
|
+
}
|
|
140
|
+
//# sourceMappingURL=migrate-local.js.map
|
|
@@ -0,0 +1,311 @@
|
|
|
1
|
+
// `vidfarm sync` — move files between the LOCAL disk backend (what `vidfarm
|
|
2
|
+
// serve` + the file directory read) and the CLOUD (vidfarm.cc), with filepath
|
|
3
|
+
// conflict resolution. This is how an AI agent hands local work up to the cloud
|
|
4
|
+
// after a long local-first editing / clip-hunting session:
|
|
5
|
+
//
|
|
6
|
+
// vidfarm sync push local My Files → cloud (upload local work)
|
|
7
|
+
// vidfarm sync pull cloud My Files → local (bring cloud assets down)
|
|
8
|
+
// vidfarm sync push /files/brand scope to a subtree
|
|
9
|
+
// --dry-run list the deltas + conflicts, transfer nothing
|
|
10
|
+
// --on-conflict newest|skip|keep-both|local|cloud (default: prompt; agents = newest)
|
|
11
|
+
//
|
|
12
|
+
// Scope: the /files (My Files) root — the durable, vector-searchable per-user
|
|
13
|
+
// asset store. Both sides share the canonical /files path, so "same path" is the
|
|
14
|
+
// collision key. Raws (/raws) sync is a documented follow-up (it needs the
|
|
15
|
+
// source-ingest pipeline, not a plain byte copy).
|
|
16
|
+
import { parseArgs } from "node:util";
|
|
17
|
+
import { homedir } from "node:os";
|
|
18
|
+
import path from "node:path";
|
|
19
|
+
import { createInterface } from "node:readline";
|
|
20
|
+
const GREEN = "\x1b[32m";
|
|
21
|
+
const YELLOW = "\x1b[33m";
|
|
22
|
+
const RED = "\x1b[31m";
|
|
23
|
+
const DIM = "\x1b[2m";
|
|
24
|
+
const BOLD = "\x1b[1m";
|
|
25
|
+
const RESET = "\x1b[0m";
|
|
26
|
+
const DEFAULT_HOST = "https://vidfarm.cc";
|
|
27
|
+
function authHeaders(apiKey) {
|
|
28
|
+
const h = { accept: "application/json" };
|
|
29
|
+
if (apiKey)
|
|
30
|
+
h["vidfarm-api-key"] = apiKey;
|
|
31
|
+
return h;
|
|
32
|
+
}
|
|
33
|
+
// ── transport: one request/upload/delete per space ──────────────────────────
|
|
34
|
+
async function req(ctx, space, input) {
|
|
35
|
+
if (space === "local") {
|
|
36
|
+
const { localApiRequest } = await import("./local-backend.js");
|
|
37
|
+
const r = await localApiRequest({ ...input, auth: { apiKey: ctx.apiKey }, home: ctx.home });
|
|
38
|
+
return { status: r.status, ok: r.ok, json: r.json };
|
|
39
|
+
}
|
|
40
|
+
const url = new URL(input.path, ctx.host);
|
|
41
|
+
for (const [k, v] of Object.entries(input.query ?? {}))
|
|
42
|
+
if (v)
|
|
43
|
+
url.searchParams.set(k, v);
|
|
44
|
+
const headers = authHeaders(ctx.apiKey);
|
|
45
|
+
let body;
|
|
46
|
+
if (input.body !== undefined) {
|
|
47
|
+
headers["content-type"] = "application/json";
|
|
48
|
+
body = JSON.stringify(input.body);
|
|
49
|
+
}
|
|
50
|
+
const res = await fetch(url, { method: input.method.toUpperCase(), headers, body });
|
|
51
|
+
const text = await res.text();
|
|
52
|
+
let json = null;
|
|
53
|
+
try {
|
|
54
|
+
json = text ? JSON.parse(text) : null;
|
|
55
|
+
}
|
|
56
|
+
catch {
|
|
57
|
+
json = null;
|
|
58
|
+
}
|
|
59
|
+
return { status: res.status, ok: res.ok, json };
|
|
60
|
+
}
|
|
61
|
+
/** Fetch the raw bytes of a file item from its own space. */
|
|
62
|
+
async function readBytes(ctx, space, item) {
|
|
63
|
+
if (!item.viewUrl)
|
|
64
|
+
throw new Error(`No view URL for ${item.path}`);
|
|
65
|
+
if (space === "local") {
|
|
66
|
+
const { withLocalBackend } = await import("./local-backend.js");
|
|
67
|
+
const backend = await withLocalBackend({ home: ctx.home, apiKey: ctx.apiKey });
|
|
68
|
+
const u = new URL(item.viewUrl, "http://vidfarm.local");
|
|
69
|
+
const res = await backend.app.request(u.pathname + u.search, { headers: authHeaders(ctx.apiKey) });
|
|
70
|
+
if (!res.ok)
|
|
71
|
+
throw new Error(`local read ${item.path} → HTTP ${res.status}`);
|
|
72
|
+
return Buffer.from(await res.arrayBuffer());
|
|
73
|
+
}
|
|
74
|
+
const res = await fetch(item.viewUrl, { headers: authHeaders(ctx.apiKey) });
|
|
75
|
+
if (!res.ok)
|
|
76
|
+
throw new Error(`cloud read ${item.path} → HTTP ${res.status}`);
|
|
77
|
+
return Buffer.from(await res.arrayBuffer());
|
|
78
|
+
}
|
|
79
|
+
/** Upload bytes into a space's My Files at folderPath/fileName (multipart — the
|
|
80
|
+
* one transport that works on both S3 and local storage). */
|
|
81
|
+
async function uploadBytes(ctx, space, input) {
|
|
82
|
+
const form = new FormData();
|
|
83
|
+
const blob = new Blob([new Uint8Array(input.bytes)], { type: input.contentType || "application/octet-stream" });
|
|
84
|
+
form.set("file", blob, input.fileName);
|
|
85
|
+
if (input.folderPath)
|
|
86
|
+
form.set("folder_path", input.folderPath);
|
|
87
|
+
if (input.notes)
|
|
88
|
+
form.set("notes", input.notes);
|
|
89
|
+
const p = "/api/v1/user/me/attachments/upload";
|
|
90
|
+
if (space === "local") {
|
|
91
|
+
const { withLocalBackend } = await import("./local-backend.js");
|
|
92
|
+
const backend = await withLocalBackend({ home: ctx.home, apiKey: ctx.apiKey });
|
|
93
|
+
const res = await backend.app.request(p, { method: "POST", headers: authHeaders(ctx.apiKey), body: form });
|
|
94
|
+
if (!res.ok)
|
|
95
|
+
throw new Error(`local upload ${input.folderPath}/${input.fileName} → HTTP ${res.status}`);
|
|
96
|
+
return;
|
|
97
|
+
}
|
|
98
|
+
const res = await fetch(new URL(p, ctx.host), { method: "POST", headers: authHeaders(ctx.apiKey), body: form });
|
|
99
|
+
if (!res.ok)
|
|
100
|
+
throw new Error(`cloud upload ${input.folderPath}/${input.fileName} → HTTP ${res.status} ${(await res.text()).slice(0, 200)}`);
|
|
101
|
+
}
|
|
102
|
+
async function deleteAttachment(ctx, space, id) {
|
|
103
|
+
await req(ctx, space, { method: "DELETE", path: `/api/v1/user/me/attachments/${encodeURIComponent(id)}` }).catch(() => undefined);
|
|
104
|
+
}
|
|
105
|
+
// ── enumerate a root recursively as a flat file list ────────────────────────
|
|
106
|
+
function toFileItem(raw) {
|
|
107
|
+
const meta = raw?.meta ?? {};
|
|
108
|
+
const updatedAtRaw = meta.updatedAt ?? meta.updated_at ?? raw.updatedAt ?? raw.createdAt ?? meta.createdAt;
|
|
109
|
+
const updatedAt = updatedAtRaw ? Date.parse(String(updatedAtRaw)) : undefined;
|
|
110
|
+
return {
|
|
111
|
+
path: String(raw?.path ?? ""),
|
|
112
|
+
name: String(raw?.name ?? ""),
|
|
113
|
+
folderPath: String(raw?.folderPath ?? ""),
|
|
114
|
+
id: raw?.id,
|
|
115
|
+
contentType: raw?.contentType,
|
|
116
|
+
sizeBytes: typeof raw?.sizeBytes === "number" ? raw.sizeBytes : undefined,
|
|
117
|
+
viewUrl: raw?.viewUrl,
|
|
118
|
+
updatedAt: Number.isFinite(updatedAt) ? updatedAt : undefined,
|
|
119
|
+
notes: typeof meta.notes === "string" ? meta.notes : undefined
|
|
120
|
+
};
|
|
121
|
+
}
|
|
122
|
+
async function enumerateFiles(ctx, space, rootPath) {
|
|
123
|
+
const out = [];
|
|
124
|
+
const queue = [rootPath];
|
|
125
|
+
const seen = new Set();
|
|
126
|
+
while (queue.length) {
|
|
127
|
+
const dir = queue.shift();
|
|
128
|
+
if (seen.has(dir))
|
|
129
|
+
continue;
|
|
130
|
+
seen.add(dir);
|
|
131
|
+
const r = await req(ctx, space, { method: "GET", path: "/api/v1/user/me/directory", query: { path: dir, limit: "500" } });
|
|
132
|
+
if (!r.ok)
|
|
133
|
+
throw new Error(`${space} directory ls ${dir} → HTTP ${r.status}${r.json?.error ? ` (${r.json.error})` : ""}`);
|
|
134
|
+
for (const f of Array.isArray(r.json?.folders) ? r.json.folders : []) {
|
|
135
|
+
if (f?.path)
|
|
136
|
+
queue.push(String(f.path));
|
|
137
|
+
}
|
|
138
|
+
for (const f of Array.isArray(r.json?.files) ? r.json.files : []) {
|
|
139
|
+
out.push(toFileItem(f));
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
return out;
|
|
143
|
+
}
|
|
144
|
+
async function resolveConflict(policy, src, dst, direction) {
|
|
145
|
+
const srcIsLocal = direction === "push";
|
|
146
|
+
switch (policy) {
|
|
147
|
+
case "skip":
|
|
148
|
+
return "skip";
|
|
149
|
+
case "keep-both":
|
|
150
|
+
return "keep-both";
|
|
151
|
+
case "local":
|
|
152
|
+
return srcIsLocal ? "transfer" : "skip";
|
|
153
|
+
case "cloud":
|
|
154
|
+
return srcIsLocal ? "skip" : "transfer";
|
|
155
|
+
case "newest": {
|
|
156
|
+
const s = src.updatedAt ?? 0;
|
|
157
|
+
const d = dst.updatedAt ?? 0;
|
|
158
|
+
if (s === d)
|
|
159
|
+
return "skip";
|
|
160
|
+
return s > d ? "transfer" : "skip";
|
|
161
|
+
}
|
|
162
|
+
case "prompt":
|
|
163
|
+
default: {
|
|
164
|
+
if (!process.stdin.isTTY) {
|
|
165
|
+
// Non-interactive (AI agent): newest-wins.
|
|
166
|
+
return resolveConflict("newest", src, dst, direction);
|
|
167
|
+
}
|
|
168
|
+
const answer = await prompt(` ${YELLOW}conflict${RESET} ${src.path}\n keep [${BOLD}s${RESET}ource=${srcIsLocal ? "local" : "cloud"}] · [${BOLD}k${RESET}eep target] · [${BOLD}b${RESET}oth] · [s${BOLD}K${RESET}ip]? `);
|
|
169
|
+
const a = answer.trim().toLowerCase();
|
|
170
|
+
if (a === "s" || a === "source")
|
|
171
|
+
return "transfer";
|
|
172
|
+
if (a === "b" || a === "both")
|
|
173
|
+
return "keep-both";
|
|
174
|
+
return "skip";
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
function prompt(q) {
|
|
179
|
+
const rl = createInterface({ input: process.stdin, output: process.stdout });
|
|
180
|
+
return new Promise((resolve) => rl.question(q, (a) => { rl.close(); resolve(a); }));
|
|
181
|
+
}
|
|
182
|
+
// Suffix a file name before its extension: "logo.png" → "logo (cloud).png".
|
|
183
|
+
function suffixName(name, suffix) {
|
|
184
|
+
const dot = name.lastIndexOf(".");
|
|
185
|
+
if (dot <= 0)
|
|
186
|
+
return `${name} (${suffix})`;
|
|
187
|
+
return `${name.slice(0, dot)} (${suffix})${name.slice(dot)}`;
|
|
188
|
+
}
|
|
189
|
+
// ── command ─────────────────────────────────────────────────────────────────
|
|
190
|
+
export async function runSyncCommand(argv) {
|
|
191
|
+
const direction = argv[0];
|
|
192
|
+
if (direction !== "push" && direction !== "pull") {
|
|
193
|
+
console.log(SYNC_HELP);
|
|
194
|
+
if (direction && direction !== "help" && direction !== "--help" && direction !== "-h")
|
|
195
|
+
process.exitCode = 1;
|
|
196
|
+
return;
|
|
197
|
+
}
|
|
198
|
+
const { values, positionals } = parseArgs({
|
|
199
|
+
args: argv.slice(1),
|
|
200
|
+
allowPositionals: true,
|
|
201
|
+
options: {
|
|
202
|
+
host: { type: "string", default: DEFAULT_HOST },
|
|
203
|
+
"api-key": { type: "string" },
|
|
204
|
+
home: { type: "string" },
|
|
205
|
+
"dry-run": { type: "boolean", default: false },
|
|
206
|
+
"on-conflict": { type: "string" }
|
|
207
|
+
}
|
|
208
|
+
});
|
|
209
|
+
const policy = normalizePolicy(values["on-conflict"]);
|
|
210
|
+
const ctx = {
|
|
211
|
+
host: String(values.host ?? DEFAULT_HOST).replace(/\/+$/, ""),
|
|
212
|
+
apiKey: values["api-key"] ?? process.env.VIDFARM_API_KEY,
|
|
213
|
+
home: values.home
|
|
214
|
+
};
|
|
215
|
+
// Scope: a /files subtree (default the whole root).
|
|
216
|
+
let rootPath = (positionals[0] ?? "/files").trim();
|
|
217
|
+
if (!rootPath.startsWith("/files")) {
|
|
218
|
+
throw new Error(`sync currently covers the /files (My Files) root — got "${rootPath}". Raws sync is a follow-up.`);
|
|
219
|
+
}
|
|
220
|
+
const srcSpace = direction === "push" ? "local" : "cloud";
|
|
221
|
+
const dstSpace = direction === "push" ? "cloud" : "local";
|
|
222
|
+
const homeLabel = ctx.home ?? process.env.VIDFARM_HOME ?? path.join(homedir(), ".vidfarm");
|
|
223
|
+
console.log(`${BOLD}sync ${direction}${RESET} ${DIM}${srcSpace} → ${dstSpace}${RESET} ${DIM}scope=${rootPath} · local=${homeLabel} · cloud=${ctx.host} · on-conflict=${policy}${values["dry-run"] ? " · DRY RUN" : ""}${RESET}\n`);
|
|
224
|
+
const [srcFiles, dstFiles] = await Promise.all([enumerateFiles(ctx, srcSpace, rootPath), enumerateFiles(ctx, dstSpace, rootPath)]);
|
|
225
|
+
const dstByPath = new Map(dstFiles.map((f) => [f.path, f]));
|
|
226
|
+
let transferred = 0;
|
|
227
|
+
let skipped = 0;
|
|
228
|
+
let conflicts = 0;
|
|
229
|
+
let inSync = 0;
|
|
230
|
+
for (const src of srcFiles) {
|
|
231
|
+
const dst = dstByPath.get(src.path);
|
|
232
|
+
if (!dst) {
|
|
233
|
+
// New file on the target → transfer.
|
|
234
|
+
if (values["dry-run"]) {
|
|
235
|
+
console.log(` ${GREEN}+ new${RESET} ${src.path}`);
|
|
236
|
+
}
|
|
237
|
+
else {
|
|
238
|
+
await transfer(ctx, srcSpace, dstSpace, src, src.name);
|
|
239
|
+
console.log(` ${GREEN}✓ sent${RESET} ${src.path}`);
|
|
240
|
+
}
|
|
241
|
+
transferred += 1;
|
|
242
|
+
continue;
|
|
243
|
+
}
|
|
244
|
+
// Same path both sides. Equal size → treat as already in sync (cheap, no
|
|
245
|
+
// byte compare); differing size → a real conflict.
|
|
246
|
+
if (src.sizeBytes != null && dst.sizeBytes != null && src.sizeBytes === dst.sizeBytes) {
|
|
247
|
+
inSync += 1;
|
|
248
|
+
continue;
|
|
249
|
+
}
|
|
250
|
+
conflicts += 1;
|
|
251
|
+
const resolution = await resolveConflict(policy, src, dst, direction);
|
|
252
|
+
if (resolution === "skip") {
|
|
253
|
+
console.log(` ${DIM}– skip${RESET} ${src.path} ${DIM}(conflict)${RESET}`);
|
|
254
|
+
skipped += 1;
|
|
255
|
+
continue;
|
|
256
|
+
}
|
|
257
|
+
if (values["dry-run"]) {
|
|
258
|
+
console.log(` ${YELLOW}~ ${resolution === "keep-both" ? "both" : "over"}${RESET} ${src.path}`);
|
|
259
|
+
transferred += 1;
|
|
260
|
+
continue;
|
|
261
|
+
}
|
|
262
|
+
if (resolution === "keep-both") {
|
|
263
|
+
await transfer(ctx, srcSpace, dstSpace, src, suffixName(src.name, srcSpace));
|
|
264
|
+
console.log(` ${YELLOW}✓ both${RESET} ${src.path} ${DIM}→ ${suffixName(src.name, srcSpace)}${RESET}`);
|
|
265
|
+
}
|
|
266
|
+
else {
|
|
267
|
+
// Overwrite: drop the target copy first so we replace rather than dup.
|
|
268
|
+
if (dst.id)
|
|
269
|
+
await deleteAttachment(ctx, dstSpace, dst.id);
|
|
270
|
+
await transfer(ctx, srcSpace, dstSpace, src, src.name);
|
|
271
|
+
console.log(` ${GREEN}✓ over${RESET} ${src.path}`);
|
|
272
|
+
}
|
|
273
|
+
transferred += 1;
|
|
274
|
+
}
|
|
275
|
+
console.log(`\n ${transferred} transferred · ${inSync} already in sync · ${conflicts} conflict${conflicts === 1 ? "" : "s"} · ${skipped} skipped`);
|
|
276
|
+
if (values["dry-run"])
|
|
277
|
+
console.log(` ${DIM}(dry run — nothing was written)${RESET}`);
|
|
278
|
+
}
|
|
279
|
+
async function transfer(ctx, from, to, item, fileName) {
|
|
280
|
+
const bytes = await readBytes(ctx, from, item);
|
|
281
|
+
await uploadBytes(ctx, to, {
|
|
282
|
+
fileName,
|
|
283
|
+
folderPath: item.folderPath.replace(/^files\/?/, "").replace(/^\/+|\/+$/g, ""),
|
|
284
|
+
contentType: item.contentType,
|
|
285
|
+
notes: item.notes,
|
|
286
|
+
bytes
|
|
287
|
+
});
|
|
288
|
+
}
|
|
289
|
+
function normalizePolicy(v) {
|
|
290
|
+
const s = String(v ?? "").trim().toLowerCase();
|
|
291
|
+
if (["newest", "skip", "keep-both", "local", "cloud"].includes(s))
|
|
292
|
+
return s;
|
|
293
|
+
if (s === "keepboth" || s === "both")
|
|
294
|
+
return "keep-both";
|
|
295
|
+
return "prompt";
|
|
296
|
+
}
|
|
297
|
+
const SYNC_HELP = `vidfarm sync — move My Files between the local disk backend and the cloud
|
|
298
|
+
|
|
299
|
+
sync push [path] local → cloud (upload local work; default scope /files)
|
|
300
|
+
sync pull [path] cloud → local (bring cloud assets down)
|
|
301
|
+
|
|
302
|
+
--dry-run list deltas + conflicts, transfer nothing
|
|
303
|
+
--on-conflict <policy> newest | skip | keep-both | local | cloud
|
|
304
|
+
(default: prompt; non-interactive/agent = newest)
|
|
305
|
+
--host <url> cloud host (default https://vidfarm.cc)
|
|
306
|
+
--api-key <key> or VIDFARM_API_KEY
|
|
307
|
+
--home <dir> local backend home (or VIDFARM_HOME; default ~/.vidfarm)
|
|
308
|
+
|
|
309
|
+
Collision key = the canonical /files path. Equal size ⇒ already in sync.
|
|
310
|
+
Raws (/raws) sync is a follow-up — it needs the source-ingest pipeline.`;
|
|
311
|
+
//# sourceMappingURL=sync.js.map
|