@gentbajko/slopify 3.13.2 → 3.14.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/extension/slopify-studio-chrome.zip +0 -0
- package/dist/extension/slopify-studio-firefox.zip +0 -0
- package/dist/kernel/db/migrations/0054-project-places.sql +27 -0
- package/dist/kernel/paths.js +4 -0
- package/dist/main.js +35 -2
- package/dist/patch-notes/3.14.0.md +12 -0
- package/dist/patch-notes/3.14.1.md +5 -0
- package/dist/patch-notes/index.json +12 -0
- package/dist/slices/loudness/loudnorm.js +5 -0
- package/dist/slices/onboarding/copy.js +6 -1
- package/dist/slices/storage/arrange.js +456 -0
- package/dist/slices/storage/backup-export.js +10 -3
- package/dist/slices/storage/delete-project.js +3 -0
- package/dist/slices/storage/layout.js +4 -2
- package/dist/slices/storage/places.js +171 -0
- package/dist/slices/storage/reconcile.js +54 -5
- package/dist/slices/storage/trim.js +6 -4
- package/dist/tutorials/Where-Your-Files-Live.md +19 -1
- package/package.json +1 -1
|
Binary file
|
|
Binary file
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
-- Readable project folders (3.14.0). A project's folder was its id and its files sat under
|
|
2
|
+
-- assets/<asset id>/, which nobody could find their way around. Each project now gets a
|
|
3
|
+
-- folder named after its title, and each file a place in it: Upload/ (the current version's
|
|
4
|
+
-- publishable files), Working/ (what they were made from) and History/<date>/ (older
|
|
5
|
+
-- versions). Every stored path (`assets/<id>/video.mp4`) stays as it is in every table and
|
|
6
|
+
-- archive; these two tables say where it is on disk (`slices/storage/places.ts`).
|
|
7
|
+
--
|
|
8
|
+
-- `state` is 'moving' between recording a move and finishing it, so a crash in between is
|
|
9
|
+
-- settled at the next start from what is on disk (`recoverPlaces`). `previous` is where the
|
|
10
|
+
-- file or folder was before, for that check.
|
|
11
|
+
CREATE TABLE project_folders (
|
|
12
|
+
project_id TEXT PRIMARY KEY REFERENCES projects(id) ON DELETE CASCADE,
|
|
13
|
+
folder TEXT NOT NULL CHECK(length(folder) > 0),
|
|
14
|
+
previous TEXT,
|
|
15
|
+
state TEXT NOT NULL DEFAULT 'placed' CHECK(state IN ('moving','placed')),
|
|
16
|
+
arranged TEXT
|
|
17
|
+
);
|
|
18
|
+
CREATE UNIQUE INDEX project_folders_folder ON project_folders(folder COLLATE NOCASE);
|
|
19
|
+
CREATE TABLE file_places (
|
|
20
|
+
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
21
|
+
path TEXT NOT NULL,
|
|
22
|
+
place TEXT NOT NULL,
|
|
23
|
+
previous TEXT,
|
|
24
|
+
state TEXT NOT NULL DEFAULT 'placed' CHECK(state IN ('moving','placed')),
|
|
25
|
+
PRIMARY KEY(project_id, path)
|
|
26
|
+
);
|
|
27
|
+
CREATE UNIQUE INDEX file_places_place ON file_places(project_id, place COLLATE NOCASE);
|
package/dist/kernel/paths.js
CHANGED
|
@@ -14,6 +14,10 @@ export function layout(dataDir, files) {
|
|
|
14
14
|
lock: join(root, ".lock"),
|
|
15
15
|
};
|
|
16
16
|
}
|
|
17
|
+
// Gives a running install's Paths the places of moved project folders and files.
|
|
18
|
+
export function attachPlaces(paths, places) {
|
|
19
|
+
Object.assign(paths, { places });
|
|
20
|
+
}
|
|
17
21
|
// Points a running install's Paths at moved files. The one object every slice was handed is
|
|
18
22
|
// changed in place, so the next file each one reads or writes is in the new place.
|
|
19
23
|
export function repoint(paths, files) {
|
package/dist/main.js
CHANGED
|
@@ -29,7 +29,7 @@ import { causedBy } from "./kernel/errors.js";
|
|
|
29
29
|
import { ulidIds } from "./kernel/ids.js";
|
|
30
30
|
import { acquireInstanceLock } from "./kernel/lock.js";
|
|
31
31
|
import { openLog } from "./kernel/log.js";
|
|
32
|
-
import { ensureDataDirs, ensureDirs, layout, repoint, subtitleModelDir } from "./kernel/paths.js";
|
|
32
|
+
import { attachPlaces, ensureDataDirs, ensureDirs, layout, repoint, subtitleModelDir, } from "./kernel/paths.js";
|
|
33
33
|
import { stageKinds } from "./kernel/pipeline.js";
|
|
34
34
|
import { sqliteAttempts } from "./kernel/runner/attempt-repo.js";
|
|
35
35
|
import { auditionVoice } from "./kernel/runner/audition.js";
|
|
@@ -71,9 +71,12 @@ import { createScheduleRunner } from "./slices/schedules/scheduler.js";
|
|
|
71
71
|
import { nodeCliProbe } from "./slices/settings/cli-status.js";
|
|
72
72
|
import { isLocalCliProvider, localCliConcurrency } from "./slices/settings/model.js";
|
|
73
73
|
import { providerStatuses } from "./slices/settings/readiness.js";
|
|
74
|
+
import { readSetting } from "./slices/settings/repo.js";
|
|
75
|
+
import { arrangeChanged } from "./slices/storage/arrange.js";
|
|
74
76
|
import { busyProjects } from "./slices/storage/backup-export.js";
|
|
75
77
|
import { documentsDir, nodeDocumentsHost } from "./slices/storage/documents.js";
|
|
76
|
-
import { createFilesService, ensureFilesFolders, filesLayoutOf, settleFilesLocation, } from "./slices/storage/files-location.js";
|
|
78
|
+
import { createFilesService, ensureFilesFolders, filesLayoutOf, filesMoveKey, settleFilesLocation, } from "./slices/storage/files-location.js";
|
|
79
|
+
import { createPlaces, recoverPlaces } from "./slices/storage/places.js";
|
|
77
80
|
import { reconcileStorage } from "./slices/storage/reconcile.js";
|
|
78
81
|
import { previewPictures } from "./slices/style-preview/images.js";
|
|
79
82
|
import { ffmpegStylePreview } from "./slices/style-preview/render.js";
|
|
@@ -183,6 +186,11 @@ export async function boot(config, options = {}) {
|
|
|
183
186
|
ensureDirs(paths, { mode: 0o700 });
|
|
184
187
|
else
|
|
185
188
|
await ensureFilesFolders(files);
|
|
189
|
+
// Readable project folders (`slices/storage/places.ts`): settle a move a crash cut short,
|
|
190
|
+
// then every file is looked up where it is, before anything reads one.
|
|
191
|
+
recoverPlaces(db, paths);
|
|
192
|
+
const places = createPlaces(db);
|
|
193
|
+
attachPlaces(paths, places);
|
|
186
194
|
const runtimeDb = db;
|
|
187
195
|
const interrupted = markInterruptedStages(db, clock);
|
|
188
196
|
recoverCheckpointWork(db);
|
|
@@ -298,6 +306,9 @@ export async function boot(config, options = {}) {
|
|
|
298
306
|
throw new Error("The server is not ready.");
|
|
299
307
|
};
|
|
300
308
|
let listeningPort = config.port;
|
|
309
|
+
// Folders are renamed only by a version that is committed: an update on trial can still be
|
|
310
|
+
// rolled back to a version that knows only id-named folders.
|
|
311
|
+
let arrangeAllowed = !pendingActivation;
|
|
301
312
|
const updater = createUpdater({
|
|
302
313
|
...(isUpdateToken(candidateToken)
|
|
303
314
|
? {
|
|
@@ -308,6 +319,7 @@ export async function boot(config, options = {}) {
|
|
|
308
319
|
? updateCommitted(paths.dataDir, version, candidateToken)
|
|
309
320
|
: dockerActivationCommitted(dockerState, candidateToken),
|
|
310
321
|
settle: () => {
|
|
322
|
+
arrangeAllowed = true;
|
|
311
323
|
try {
|
|
312
324
|
const settled = reconcileStorage(updateDb, paths);
|
|
313
325
|
log.write("info", "update", {
|
|
@@ -623,6 +635,26 @@ export async function boot(config, options = {}) {
|
|
|
623
635
|
};
|
|
624
636
|
purgeTrash();
|
|
625
637
|
const trashTimer = setInterval(purgeTrash, 60 * 60_000);
|
|
638
|
+
// Each project whose title, version or files changed is arranged into its readable folder
|
|
639
|
+
// (`slices/storage/arrange.ts`) once nothing of it runs: a quarter second of it at a time,
|
|
640
|
+
// so the server keeps answering, and never while files move or an update installs.
|
|
641
|
+
const arrange = () => {
|
|
642
|
+
if (!arrangeAllowed || readSetting(updateDb, filesMoveKey) !== undefined)
|
|
643
|
+
return;
|
|
644
|
+
const release = updater.beginMutation();
|
|
645
|
+
if (!release)
|
|
646
|
+
return;
|
|
647
|
+
try {
|
|
648
|
+
arrangeChanged({ db: updateDb, paths, places, now: () => clock.now() }, 250);
|
|
649
|
+
}
|
|
650
|
+
catch (error) {
|
|
651
|
+
log.write("error", "storage.arrange", { detail: causedBy(error) });
|
|
652
|
+
}
|
|
653
|
+
finally {
|
|
654
|
+
release();
|
|
655
|
+
}
|
|
656
|
+
};
|
|
657
|
+
const arrangeTimer = setInterval(arrange, 5_000);
|
|
626
658
|
listeningPort = portOf(server) ?? config.port;
|
|
627
659
|
// Whatever last run left queued goes out at start. Nothing waits for
|
|
628
660
|
// it, and an unreachable collector costs one refused socket.
|
|
@@ -651,6 +683,7 @@ export async function boot(config, options = {}) {
|
|
|
651
683
|
clearInterval(retryTimer);
|
|
652
684
|
clearInterval(backupTimer);
|
|
653
685
|
clearInterval(trashTimer);
|
|
686
|
+
clearInterval(arrangeTimer);
|
|
654
687
|
clearInterval(modelTimer);
|
|
655
688
|
const mutationDrain = mutations.stop();
|
|
656
689
|
const scheduleDrain = scheduleTicks.stop();
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
# Slopify 3.14.0
|
|
2
|
+
|
|
3
|
+
Released 9 October 2026.
|
|
4
|
+
|
|
5
|
+
- **Project folders you can find your way around.** Each project's folder is now named after its title instead of a code, and its files are sorted the way you look for them:
|
|
6
|
+
- `Upload/`: the current version, ready to publish: the video, thumbnails, subtitles, description, tags, titles, pinned comment, PDF, article, and `Shorts/1 - <title>.mp4`.
|
|
7
|
+
- `Working/`: what that version was made from: images, each narration chunk, research, subtitle timing, render settings.
|
|
8
|
+
- `History/<date>/`: older versions (an earlier render, the narration before a voice change). Delete a day's folder, or all of History, to get the space back; going back to that version makes its files again.
|
|
9
|
+
|
|
10
|
+
Slopify sorts your existing projects in the background after the update, a few at a time, without stopping the app; a project is sorted once nothing of it is running. Nothing is copied, so it takes no extra disk space, and your backups work in every version as before. Files you add to these folders yourself are never removed, and a folder in Projects that Slopify didn't make is left alone. See [Where Your Files Live](https://github.com/gentbajko/slopify/wiki/Where-Your-Files-Live#inside-a-project-folder).
|
|
11
|
+
|
|
12
|
+
**Don't go back to a version older than 3.14 after this update:** older versions only know folders named by project code and would remove the readable ones when they start.
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
# Slopify 3.14.1
|
|
2
|
+
|
|
3
|
+
Released 9 October 2026.
|
|
4
|
+
|
|
5
|
+
- **Videos finish sooner.** After rendering a video, Slopify checks how loud it came out. That check decoded the whole picture just to read the sound, which took 20 minutes or more for a two-hour video and slowed down any other render running at the same time. It now reads only the sound, so a video is done a few minutes after its render reaches 100%.
|
|
@@ -1,4 +1,16 @@
|
|
|
1
1
|
[
|
|
2
|
+
{
|
|
3
|
+
"id": "3.14.1",
|
|
4
|
+
"title": "Slopify 3.14.1",
|
|
5
|
+
"version": "3.14.1",
|
|
6
|
+
"date": "2026-10-09"
|
|
7
|
+
},
|
|
8
|
+
{
|
|
9
|
+
"id": "3.14.0",
|
|
10
|
+
"title": "Slopify 3.14.0",
|
|
11
|
+
"version": "3.14.0",
|
|
12
|
+
"date": "2026-10-09"
|
|
13
|
+
},
|
|
2
14
|
{
|
|
3
15
|
"id": "3.13.2",
|
|
4
16
|
"title": "Slopify 3.13.2",
|
|
@@ -25,6 +25,11 @@ export function measureArgs(input, goal, chain = "") {
|
|
|
25
25
|
"info",
|
|
26
26
|
"-nostats",
|
|
27
27
|
...input,
|
|
28
|
+
// Only the sound is measured: a finished video's picture was decoded too, 20 minutes and
|
|
29
|
+
// more of a CPU's time for a two-hour export, while the next render waited for it.
|
|
30
|
+
"-vn",
|
|
31
|
+
"-sn",
|
|
32
|
+
"-dn",
|
|
28
33
|
"-af",
|
|
29
34
|
`${chain === "" ? "" : `${chain},`}${measureFilter(goal)}`,
|
|
30
35
|
"-f",
|
|
@@ -4,6 +4,7 @@ import { transact } from "../../kernel/db/tx.js";
|
|
|
4
4
|
import { executionPlan } from "../rebuild/runtime-plan.js";
|
|
5
5
|
import { currentRevisionId } from "../revisions/repo.js";
|
|
6
6
|
import { getRevisionView } from "../revisions/view.js";
|
|
7
|
+
import { projectMarker } from "../storage/arrange.js";
|
|
7
8
|
import { projectRows } from "../storage/backup-export.js";
|
|
8
9
|
import { projectTables } from "../storage/backup-format.js";
|
|
9
10
|
import { projectDir } from "../storage/layout.js";
|
|
@@ -34,7 +35,11 @@ export function copyProject(deps, sourceId, title) {
|
|
|
34
35
|
const at = deps.clock.now().toISOString();
|
|
35
36
|
try {
|
|
36
37
|
for (const file of listFiles(source)) {
|
|
37
|
-
|
|
38
|
+
if (file === projectMarker)
|
|
39
|
+
continue;
|
|
40
|
+
// The copy is a new project, laid out by its stored paths until it is arranged.
|
|
41
|
+
const stored = deps.paths.places?.storedAt(sourceId, file) ?? file;
|
|
42
|
+
const to = join(target, remap(stored));
|
|
38
43
|
mkdirSync(dirname(to), { recursive: true, mode: 0o700 });
|
|
39
44
|
copyFileSync(join(source, file), to);
|
|
40
45
|
}
|
|
@@ -0,0 +1,456 @@
|
|
|
1
|
+
import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmdirSync, statSync, writeFileSync, } from "node:fs";
|
|
2
|
+
import { basename, dirname, extname, join } from "node:path";
|
|
3
|
+
import { pickedShortsOf } from "../shorts/clips.js";
|
|
4
|
+
import { backupsFolderName, projectDir, renderCacheFolder } from "./layout.js";
|
|
5
|
+
// The file in each project folder that says which project it is, for when the database and
|
|
6
|
+
// the folders disagree (a restored database, a folder renamed by hand). Reconcile keeps it.
|
|
7
|
+
export const projectMarker = ".slopify-project";
|
|
8
|
+
// What makes a project's arrangement current: its title, head version and files. A project
|
|
9
|
+
// whose key is unchanged since it was last arranged is left alone.
|
|
10
|
+
export function arrangeKeys(db) {
|
|
11
|
+
const keys = new Map();
|
|
12
|
+
for (const row of db
|
|
13
|
+
.prepare(`SELECT p.id, p.title, h.revision_id,
|
|
14
|
+
(SELECT count(*)||':'||COALESCE(max(a.id),'') FROM project_assets a WHERE a.project_id=p.id) AS assets,
|
|
15
|
+
(SELECT count(*)||':'||COALESCE(max(o.id),'') FROM outputs o WHERE o.project_id=p.id) AS legacy,
|
|
16
|
+
(SELECT count(*)||':'||COALESCE(max(r.id),'') FROM revision_outputs r WHERE r.project_id=p.id AND r.selected=1 AND r.revision_id=h.revision_id) AS current
|
|
17
|
+
FROM projects p LEFT JOIN project_heads h ON h.project_id=p.id`)
|
|
18
|
+
.all())
|
|
19
|
+
keys.set(String(row.id), [row.title, row.revision_id, row.assets, row.legacy, row.current].map(String).join("|"));
|
|
20
|
+
return keys;
|
|
21
|
+
}
|
|
22
|
+
// Whether anything of the project runs or may start: its files stay where they are until then.
|
|
23
|
+
export function projectSettled(db, projectId) {
|
|
24
|
+
const busy = db
|
|
25
|
+
.prepare(`SELECT 1 FROM revision_work WHERE project_id=? AND (state='running' OR (state='pending' AND dispatch_state!='held'))
|
|
26
|
+
UNION ALL SELECT 1 FROM stages WHERE project_id=? AND state='running'
|
|
27
|
+
UNION ALL SELECT 1 FROM project_trash WHERE project_id=? LIMIT 1`)
|
|
28
|
+
.get(projectId, projectId, projectId);
|
|
29
|
+
return busy === undefined;
|
|
30
|
+
}
|
|
31
|
+
export function arrangeProject(deps, projectId) {
|
|
32
|
+
const { db, paths, places } = deps;
|
|
33
|
+
if (!projectSettled(db, projectId))
|
|
34
|
+
return { folder: places.folderOf(projectId), moved: 0, skipped: "busy" };
|
|
35
|
+
const project = db.prepare("SELECT title FROM projects WHERE id=?").get(projectId);
|
|
36
|
+
if (project === undefined)
|
|
37
|
+
return { folder: undefined, moved: 0, skipped: "missing" };
|
|
38
|
+
const current = projectDir(paths, projectId);
|
|
39
|
+
if (!existsSync(current))
|
|
40
|
+
return { folder: undefined, moved: 0, skipped: "no folder" };
|
|
41
|
+
// 1. The folder, named after the title.
|
|
42
|
+
const have = places.folderOf(projectId) ?? projectId;
|
|
43
|
+
const want = folderName(String(project.title), (name) => taken(db, paths, projectId, have, name));
|
|
44
|
+
if (want !== have && want.toLowerCase() !== have.toLowerCase()) {
|
|
45
|
+
places.moves.folder(projectId, want, places.folderOf(projectId));
|
|
46
|
+
try {
|
|
47
|
+
renameSync(current, join(paths.projects, want));
|
|
48
|
+
places.moves.folderDone(projectId);
|
|
49
|
+
}
|
|
50
|
+
catch {
|
|
51
|
+
places.moves.folderUndone(projectId);
|
|
52
|
+
return { folder: have, moved: 0, skipped: "folder in use" };
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
else if (places.folderOf(projectId) === undefined) {
|
|
56
|
+
// Already readable (or the same name): still recorded, so cleanup knows it is ours.
|
|
57
|
+
places.moves.folder(projectId, have, undefined);
|
|
58
|
+
places.moves.folderDone(projectId);
|
|
59
|
+
}
|
|
60
|
+
const root = projectDir(paths, projectId);
|
|
61
|
+
// 2. Each file to its place. Files leaving for History go first, so the new version takes
|
|
62
|
+
// the names they free ("video.mp4", not "video (2).mp4").
|
|
63
|
+
const desired = [...desiredPlaces(db, projectId)].toSorted(([, a], [, b]) => Number(!a.startsWith("History/")) - Number(!b.startsWith("History/")));
|
|
64
|
+
const placed = places.placesOf(projectId);
|
|
65
|
+
let moved = 0;
|
|
66
|
+
for (const [path, base] of desired) {
|
|
67
|
+
const at = placed.get(path) ?? path;
|
|
68
|
+
if (sameBase(at, base) || !existsSync(join(root, at)))
|
|
69
|
+
continue;
|
|
70
|
+
const to = freeName(base, (name) => existsSync(join(root, name)));
|
|
71
|
+
places.moves.file(projectId, path, to, placed.get(path));
|
|
72
|
+
try {
|
|
73
|
+
mkdirSync(dirname(join(root, to)), { recursive: true });
|
|
74
|
+
renameSync(join(root, at), join(root, to));
|
|
75
|
+
places.moves.fileDone(projectId, path);
|
|
76
|
+
moved += 1;
|
|
77
|
+
}
|
|
78
|
+
catch {
|
|
79
|
+
// In use (a player or download on Windows): it stays and is tried again next time.
|
|
80
|
+
places.moves.fileUndone(projectId, path);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
removeEmptyFolders(root);
|
|
84
|
+
writeMarker(root, projectId, places.placesOf(projectId));
|
|
85
|
+
return { folder: places.folderOf(projectId), moved, skipped: undefined };
|
|
86
|
+
}
|
|
87
|
+
// Every project whose title, version or files changed since it was last arranged.
|
|
88
|
+
// Stops once `budgetMs` has passed, so a pass on the server's thread stays short; the next
|
|
89
|
+
// pass carries on where it stopped.
|
|
90
|
+
export function arrangeChanged(deps, budgetMs = Number.POSITIVE_INFINITY) {
|
|
91
|
+
const started = performance.now();
|
|
92
|
+
const done = [];
|
|
93
|
+
for (const [projectId, key] of arrangeKeys(deps.db)) {
|
|
94
|
+
if (performance.now() - started > budgetMs)
|
|
95
|
+
break;
|
|
96
|
+
if (deps.places.moves.arrangedKey(projectId) === key)
|
|
97
|
+
continue;
|
|
98
|
+
const result = arrangeProject(deps, projectId);
|
|
99
|
+
if (result.skipped === undefined) {
|
|
100
|
+
deps.places.moves.arranged(projectId, key);
|
|
101
|
+
done.push(projectId);
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
return done;
|
|
105
|
+
}
|
|
106
|
+
// ----- names -------------------------------------------------------------------------------
|
|
107
|
+
const reserved = /^(con|prn|aux|nul|com[0-9]|lpt[0-9])$/iu;
|
|
108
|
+
// A folder name every system accepts, from the project's title: "Lolth | D&D Lore To Sleep
|
|
109
|
+
// To" becomes "Lolth - D&D Lore To Sleep To".
|
|
110
|
+
export function folderName(title, isTaken) {
|
|
111
|
+
let name = title
|
|
112
|
+
.replace(/\s*[|/\\:]\s*/gu, " - ")
|
|
113
|
+
.replace(/[*?"<>\p{Cc}]/gu, "")
|
|
114
|
+
.replace(/\s+/gu, " ")
|
|
115
|
+
.trim()
|
|
116
|
+
.replace(/[. ]+$/u, "");
|
|
117
|
+
if (name.length > 90)
|
|
118
|
+
name = name.slice(0, 90).replace(/[. ]+$/u, "");
|
|
119
|
+
if (name === "" || name.startsWith("."))
|
|
120
|
+
name = `Project ${name.replace(/^\.+/u, "")}`.trim();
|
|
121
|
+
if (reserved.test(name) || name.toLowerCase() === backupsFolderName.toLowerCase())
|
|
122
|
+
name = `${name} (project)`;
|
|
123
|
+
return freeName(name, isTaken);
|
|
124
|
+
}
|
|
125
|
+
function taken(db, paths, projectId, own, name) {
|
|
126
|
+
if (name.toLowerCase() === own.toLowerCase())
|
|
127
|
+
return false;
|
|
128
|
+
if (name.toLowerCase() === renderCacheFolder)
|
|
129
|
+
return true;
|
|
130
|
+
const row = db
|
|
131
|
+
.prepare("SELECT 1 FROM project_folders WHERE folder=? COLLATE NOCASE AND project_id!=?")
|
|
132
|
+
.get(name, projectId);
|
|
133
|
+
return row !== undefined || existsSync(join(paths.projects, name)) || isId(db, name);
|
|
134
|
+
}
|
|
135
|
+
function isId(db, name) {
|
|
136
|
+
return db.prepare("SELECT 1 FROM projects WHERE id=?").get(name) !== undefined;
|
|
137
|
+
}
|
|
138
|
+
// "video.mp4", then "video (2).mp4", "video (3).mp4"…
|
|
139
|
+
function freeName(name, isTaken) {
|
|
140
|
+
if (!isTaken(name))
|
|
141
|
+
return name;
|
|
142
|
+
const extension = name.includes("/") ? extname(basename(name)) : extname(name);
|
|
143
|
+
const stem = extension === "" ? name : name.slice(0, -extension.length);
|
|
144
|
+
for (let n = 2;; n++) {
|
|
145
|
+
const next = `${stem} (${String(n)})${extension}`;
|
|
146
|
+
if (!isTaken(next))
|
|
147
|
+
return next;
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
function sameBase(at, base) {
|
|
151
|
+
if (at.toLowerCase() === base.toLowerCase())
|
|
152
|
+
return true;
|
|
153
|
+
const extension = extname(base);
|
|
154
|
+
const stem = extension === "" ? base : base.slice(0, -extension.length);
|
|
155
|
+
const pattern = new RegExp(`^${literal(stem)} \\([0-9]+\\)${literal(extension)}$`, "iu");
|
|
156
|
+
return pattern.test(at);
|
|
157
|
+
}
|
|
158
|
+
function literal(text) {
|
|
159
|
+
return text.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&");
|
|
160
|
+
}
|
|
161
|
+
function clean(text) {
|
|
162
|
+
return text
|
|
163
|
+
.replace(/[|/\\:*?"<>\p{Cc}]/gu, " ")
|
|
164
|
+
.replace(/\s+/gu, " ")
|
|
165
|
+
.trim()
|
|
166
|
+
.replace(/[. ]+$/u, "")
|
|
167
|
+
.slice(0, 80);
|
|
168
|
+
}
|
|
169
|
+
const pad = (n, width = 3) => String(n).padStart(width, "0");
|
|
170
|
+
// Stored path → its place (before any " (2)" for a clash).
|
|
171
|
+
export function desiredPlaces(db, projectId) {
|
|
172
|
+
const head = db
|
|
173
|
+
.prepare("SELECT revision_id FROM project_heads WHERE project_id=?")
|
|
174
|
+
.get(projectId);
|
|
175
|
+
const headId = head === undefined ? undefined : String(head.revision_id);
|
|
176
|
+
const content = headId === undefined
|
|
177
|
+
? undefined
|
|
178
|
+
: db.prepare("SELECT content FROM project_revisions WHERE id=?").get(headId);
|
|
179
|
+
const order = imageOrder(content?.content);
|
|
180
|
+
const titles = shortTitles(db, headId);
|
|
181
|
+
const references = new Map();
|
|
182
|
+
const add = (reference) => {
|
|
183
|
+
const old = references.get(reference.path);
|
|
184
|
+
// The current version's use names a file; otherwise its latest use.
|
|
185
|
+
if (old === undefined ||
|
|
186
|
+
(reference.current && !old.current) ||
|
|
187
|
+
(reference.current === old.current && reference.createdAt > old.createdAt))
|
|
188
|
+
references.set(reference.path, reference);
|
|
189
|
+
};
|
|
190
|
+
for (const row of db
|
|
191
|
+
.prepare(`SELECT a.path, o.slot, o.descriptor, o.created_at, (o.revision_id=? AND o.selected=1) AS current
|
|
192
|
+
FROM revision_outputs o JOIN project_assets a ON a.id=o.asset_id AND a.project_id=o.project_id
|
|
193
|
+
WHERE o.project_id=?`)
|
|
194
|
+
.all(headId ?? null, projectId)) {
|
|
195
|
+
const descriptor = parse(row.descriptor);
|
|
196
|
+
const meta = descriptor?.meta;
|
|
197
|
+
add({
|
|
198
|
+
path: String(row.path),
|
|
199
|
+
role: typeof descriptor?.role === "string" ? descriptor.role : "other",
|
|
200
|
+
slot: String(row.slot),
|
|
201
|
+
index: undefined,
|
|
202
|
+
short: typeof meta?.short === "number" ? meta.short : undefined,
|
|
203
|
+
createdAt: String(row.created_at),
|
|
204
|
+
current: row.current === 1,
|
|
205
|
+
});
|
|
206
|
+
}
|
|
207
|
+
for (const row of db
|
|
208
|
+
.prepare(`SELECT a.path, p.piece_key, p.descriptor, p.created_at, (p.revision_id=? AND p.selected=1) AS current
|
|
209
|
+
FROM revision_pieces p JOIN project_assets a ON a.id=p.asset_id AND a.project_id=p.project_id
|
|
210
|
+
WHERE p.project_id=?`)
|
|
211
|
+
.all(headId ?? null, projectId)) {
|
|
212
|
+
const descriptor = parse(row.descriptor);
|
|
213
|
+
add({
|
|
214
|
+
path: String(row.path),
|
|
215
|
+
role: `piece:${typeof descriptor?.kind === "string" ? descriptor.kind : "other"}`,
|
|
216
|
+
slot: String(row.piece_key),
|
|
217
|
+
index: typeof descriptor?.idx === "number" ? descriptor.idx : undefined,
|
|
218
|
+
short: undefined,
|
|
219
|
+
createdAt: String(row.created_at),
|
|
220
|
+
current: row.current === 1,
|
|
221
|
+
});
|
|
222
|
+
}
|
|
223
|
+
// A project from before versions: its outputs rows are its files, all current.
|
|
224
|
+
for (const row of db
|
|
225
|
+
.prepare("SELECT path, role, created_at FROM outputs WHERE project_id=?")
|
|
226
|
+
.all(projectId))
|
|
227
|
+
add({
|
|
228
|
+
path: String(row.path),
|
|
229
|
+
role: String(row.role),
|
|
230
|
+
slot: String(row.role),
|
|
231
|
+
index: undefined,
|
|
232
|
+
short: undefined,
|
|
233
|
+
createdAt: String(row.created_at),
|
|
234
|
+
current: headId === undefined,
|
|
235
|
+
});
|
|
236
|
+
const out = new Map();
|
|
237
|
+
for (const reference of references.values()) {
|
|
238
|
+
const place = placeFor(reference, order, titles);
|
|
239
|
+
out.set(reference.path, reference.current
|
|
240
|
+
? place
|
|
241
|
+
: `History/${day(reference.createdAt)}/${place.replace(/^Upload\//u, "")}`);
|
|
242
|
+
}
|
|
243
|
+
return out;
|
|
244
|
+
}
|
|
245
|
+
function placeFor(reference, order, titles) {
|
|
246
|
+
const file = basename(reference.path);
|
|
247
|
+
const extension = extname(file);
|
|
248
|
+
const slot = reference.slot;
|
|
249
|
+
const segment = /^(?:audio|level|narration:files):(body|intro|outro)/u.exec(slot)?.[1];
|
|
250
|
+
switch (reference.role) {
|
|
251
|
+
case "video":
|
|
252
|
+
return `Upload/video${extension}`;
|
|
253
|
+
case "thumbnail": {
|
|
254
|
+
const n = /^thumbnail:image:([0-9]+)$/u.exec(slot)?.[1];
|
|
255
|
+
return `Upload/thumbnail${n === undefined ? "" : ` ${n}`}${extension}`;
|
|
256
|
+
}
|
|
257
|
+
case "subtitles_srt":
|
|
258
|
+
case "subtitles_vtt":
|
|
259
|
+
return `Upload/subtitles${extension}`;
|
|
260
|
+
case "youtube_description":
|
|
261
|
+
return "Upload/description.txt";
|
|
262
|
+
case "youtube_tags":
|
|
263
|
+
return "Upload/tags.txt";
|
|
264
|
+
case "youtube_titles":
|
|
265
|
+
return "Upload/titles.txt";
|
|
266
|
+
case "youtube_pinned_comment":
|
|
267
|
+
return "Upload/pinned comment.txt";
|
|
268
|
+
case "document_pdf":
|
|
269
|
+
return `Upload/document${extension}`;
|
|
270
|
+
case "article_md":
|
|
271
|
+
return `Upload/article${extension}`;
|
|
272
|
+
case "audio_export":
|
|
273
|
+
return `Upload/audio${extension}`;
|
|
274
|
+
case "short_video": {
|
|
275
|
+
const n = reference.short ?? Number(/^shorts:([0-9]+):/u.exec(slot)?.[1] ?? Number.NaN);
|
|
276
|
+
if (Number.isNaN(n))
|
|
277
|
+
return `Upload/Shorts/${file}`;
|
|
278
|
+
const title = titles.get(n);
|
|
279
|
+
return `Upload/Shorts/${String(n)}${title === undefined || title === "" ? "" : ` - ${clean(title)}`}${extension}`;
|
|
280
|
+
}
|
|
281
|
+
case "image":
|
|
282
|
+
case "animated_image": {
|
|
283
|
+
const key = /^image:(.+)$/u.exec(slot)?.[1];
|
|
284
|
+
const at = key === undefined ? undefined : order.get(key);
|
|
285
|
+
return at === undefined
|
|
286
|
+
? `Working/Images/${file}`
|
|
287
|
+
: `Working/Images/${pad(at + 1)}${extension}`;
|
|
288
|
+
}
|
|
289
|
+
case "short_image": {
|
|
290
|
+
const match = /^shorts:([0-9]+):image:([0-9]+)$/u.exec(slot);
|
|
291
|
+
return match === null
|
|
292
|
+
? `Working/Shorts/${file}`
|
|
293
|
+
: `Working/Shorts/Short ${match[1]}/image ${match[2]}${extension}`;
|
|
294
|
+
}
|
|
295
|
+
case "figure_card":
|
|
296
|
+
return `Working/Cards/${file}`;
|
|
297
|
+
case "audio_body":
|
|
298
|
+
case "audio_intro":
|
|
299
|
+
case "audio_outro":
|
|
300
|
+
return `Working/Narration/${segment ?? reference.role.slice(6)}${extension}`;
|
|
301
|
+
case "audio_levelled":
|
|
302
|
+
return `Working/Narration/${segment ?? "narration"} levelled${extension}`;
|
|
303
|
+
case "narration_txt":
|
|
304
|
+
return `Working/Narration/${segment ?? "narration"} text${extension}`;
|
|
305
|
+
case "tts_script":
|
|
306
|
+
return `Working/Narration/${segment ?? "narration"} voice script${extension}`;
|
|
307
|
+
case "piece:chunk":
|
|
308
|
+
case "piece:segment":
|
|
309
|
+
return `Working/Narration/Chunks/${segment ?? "body"} ${pad((reference.index ?? 0) + 1)}${extension}`;
|
|
310
|
+
case "notes":
|
|
311
|
+
return `Working/Research/notes${extension}`;
|
|
312
|
+
case "piece:chapter": {
|
|
313
|
+
const n = /:([0-9]+)$/u.exec(slot)?.[1];
|
|
314
|
+
return `Working/Research/chapter ${n ?? file}${n === undefined ? "" : extension}`;
|
|
315
|
+
}
|
|
316
|
+
case "article_txt":
|
|
317
|
+
case "sources":
|
|
318
|
+
case "glossary":
|
|
319
|
+
return `Working/Article/${reference.role === "article_txt" ? "article" : reference.role}${extension}`;
|
|
320
|
+
case "piece:article_written":
|
|
321
|
+
// The joined narration's parts (audio:body:concat), or an article's written parts.
|
|
322
|
+
return segment === undefined
|
|
323
|
+
? `Working/Article/Parts/${file}`
|
|
324
|
+
: `Working/Narration/Joined/${segment} ${file}`;
|
|
325
|
+
case "instructions": {
|
|
326
|
+
const stage = /^([a-z]+):instructions$/u.exec(slot)?.[1];
|
|
327
|
+
return `Working/Instructions/${stage ?? basename(file, extension)}${extension}`;
|
|
328
|
+
}
|
|
329
|
+
case "subtitle_words":
|
|
330
|
+
case "subtitle_ass":
|
|
331
|
+
case "subtitle_font":
|
|
332
|
+
return `Working/Subtitles/${file}`;
|
|
333
|
+
case "render_params":
|
|
334
|
+
case "shorts":
|
|
335
|
+
return `Working/Render/${file}`;
|
|
336
|
+
default:
|
|
337
|
+
return `Working/Other/${file}`;
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
function imageOrder(content) {
|
|
341
|
+
const parsed = parse(content);
|
|
342
|
+
const order = Array.isArray(parsed?.imageOrder) ? parsed.imageOrder : [];
|
|
343
|
+
return new Map(order.map((key, at) => [String(key), at]));
|
|
344
|
+
}
|
|
345
|
+
function shortTitles(db, headId) {
|
|
346
|
+
if (headId === undefined)
|
|
347
|
+
return new Map();
|
|
348
|
+
const row = db
|
|
349
|
+
.prepare("SELECT descriptor FROM revision_pieces WHERE revision_id=? AND piece_key='shorts:pick' AND selected=1")
|
|
350
|
+
.get(headId);
|
|
351
|
+
const payload = parse(row?.descriptor)?.payload;
|
|
352
|
+
const picked = pickedShortsOf(typeof payload === "string" ? payload : undefined);
|
|
353
|
+
return new Map((picked?.shorts ?? []).map((short) => [short.number, short.title]));
|
|
354
|
+
}
|
|
355
|
+
function day(createdAt) {
|
|
356
|
+
const at = new Date(createdAt);
|
|
357
|
+
if (Number.isNaN(at.getTime()))
|
|
358
|
+
return "undated";
|
|
359
|
+
return `${String(at.getFullYear())}-${pad(at.getMonth() + 1, 2)}-${pad(at.getDate(), 2)}`;
|
|
360
|
+
}
|
|
361
|
+
function parse(value) {
|
|
362
|
+
if (typeof value !== "string")
|
|
363
|
+
return undefined;
|
|
364
|
+
try {
|
|
365
|
+
const parsed = JSON.parse(value);
|
|
366
|
+
return typeof parsed === "object" && parsed !== null
|
|
367
|
+
? parsed
|
|
368
|
+
: undefined;
|
|
369
|
+
}
|
|
370
|
+
catch {
|
|
371
|
+
return undefined;
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
// ----- the folder itself ---------------------------------------------------------------------
|
|
375
|
+
// The marker names the project and lists where each of its files went, so a folder can be
|
|
376
|
+
// matched back to its project and its files found again even by a database restored from
|
|
377
|
+
// before it was arranged (`adoptMarked`). Written whole, then renamed into place.
|
|
378
|
+
function writeMarker(root, projectId, files) {
|
|
379
|
+
const path = join(root, projectMarker);
|
|
380
|
+
const part = `${path}.part`;
|
|
381
|
+
writeFileSync(part, `${JSON.stringify({
|
|
382
|
+
project: projectId,
|
|
383
|
+
note: "Slopify keeps this file to know which project this folder holds and where its files are. Leave it here.",
|
|
384
|
+
files: Object.fromEntries([...files].toSorted(([a], [b]) => a.localeCompare(b))),
|
|
385
|
+
}, null, 2)}\n`, { mode: 0o600 });
|
|
386
|
+
renameSync(part, path);
|
|
387
|
+
}
|
|
388
|
+
function readMarker(dir) {
|
|
389
|
+
const path = join(dir, projectMarker);
|
|
390
|
+
if (!existsSync(path))
|
|
391
|
+
return undefined;
|
|
392
|
+
try {
|
|
393
|
+
const parsed = parse(readFileSync(path, "utf8"));
|
|
394
|
+
if (typeof parsed?.project !== "string")
|
|
395
|
+
return undefined;
|
|
396
|
+
const files = {};
|
|
397
|
+
if (typeof parsed.files === "object" && parsed.files !== null)
|
|
398
|
+
for (const [stored, place] of Object.entries(parsed.files))
|
|
399
|
+
if (typeof place === "string")
|
|
400
|
+
files[stored] = place;
|
|
401
|
+
return { project: parsed.project, files };
|
|
402
|
+
}
|
|
403
|
+
catch {
|
|
404
|
+
return undefined;
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
// The project a folder's marker names, if it has one.
|
|
408
|
+
export function markerProject(dir) {
|
|
409
|
+
return readMarker(dir)?.project;
|
|
410
|
+
}
|
|
411
|
+
// Records a marked folder, and the places of its files that are still there, for a project
|
|
412
|
+
// the database has no folder for.
|
|
413
|
+
export function adoptMarked(db, paths, projectId, folder, dir) {
|
|
414
|
+
const places = paths.places;
|
|
415
|
+
if (places === undefined || !("moves" in places))
|
|
416
|
+
return;
|
|
417
|
+
const moves = places.moves;
|
|
418
|
+
if (db.prepare("SELECT 1 FROM project_folders WHERE folder=? COLLATE NOCASE").get(folder))
|
|
419
|
+
return;
|
|
420
|
+
moves.folder(projectId, folder, undefined);
|
|
421
|
+
moves.folderDone(projectId);
|
|
422
|
+
for (const [stored, place] of Object.entries(readMarker(dir)?.files ?? {})) {
|
|
423
|
+
if (places.placeOf(projectId, stored) !== undefined)
|
|
424
|
+
continue;
|
|
425
|
+
if (!existsSync(join(dir, place)))
|
|
426
|
+
continue;
|
|
427
|
+
moves.file(projectId, stored, place, undefined);
|
|
428
|
+
moves.fileDone(projectId, stored);
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
// The empty folders moves leave behind (assets/<id>/), deepest first. The folder itself stays.
|
|
432
|
+
function removeEmptyFolders(root) {
|
|
433
|
+
const walk = (dir) => {
|
|
434
|
+
let empty = true;
|
|
435
|
+
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
|
436
|
+
const path = join(dir, entry.name);
|
|
437
|
+
if (entry.isDirectory()) {
|
|
438
|
+
if (walk(path)) {
|
|
439
|
+
try {
|
|
440
|
+
rmdirSync(path);
|
|
441
|
+
}
|
|
442
|
+
catch {
|
|
443
|
+
empty = false;
|
|
444
|
+
}
|
|
445
|
+
}
|
|
446
|
+
else
|
|
447
|
+
empty = false;
|
|
448
|
+
}
|
|
449
|
+
else
|
|
450
|
+
empty = false;
|
|
451
|
+
}
|
|
452
|
+
return empty;
|
|
453
|
+
};
|
|
454
|
+
if (existsSync(root) && statSync(root).isDirectory())
|
|
455
|
+
walk(root);
|
|
456
|
+
}
|
|
@@ -3,8 +3,9 @@ import { constants, lstatSync, readdirSync } from "node:fs";
|
|
|
3
3
|
import { open } from "node:fs/promises";
|
|
4
4
|
import { join } from "node:path";
|
|
5
5
|
import { transact } from "../../kernel/db/tx.js";
|
|
6
|
+
import { projectMarker } from "./arrange.js";
|
|
6
7
|
import { backupFormat, backupSchemaVersion, checksumsMember, fontMember, imageMember, libraryMember, manifestMember, projectFileMember, projectMember, projectTables, safeRelativePath, stagedMember, usageMember, } from "./backup-format.js";
|
|
7
|
-
import { projectDir, stagingPath } from "./layout.js";
|
|
8
|
+
import { outputPath, projectDir, stagingPath } from "./layout.js";
|
|
8
9
|
import { exportableSettings, installedUploadedFontFiles } from "./portable.js";
|
|
9
10
|
import { tarEnd, tarHeader, tarMemberBytes, tarPadding } from "./tar.js";
|
|
10
11
|
// A project that is being made has half-written files and rows that say "running"; copied
|
|
@@ -70,7 +71,7 @@ export function planBackup(deps) {
|
|
|
70
71
|
files.push({
|
|
71
72
|
name: projectFileMember(project.part.id, file.path),
|
|
72
73
|
kind: "file",
|
|
73
|
-
path:
|
|
74
|
+
path: outputPath(deps.paths, project.part.id, file.path),
|
|
74
75
|
size: file.bytes,
|
|
75
76
|
});
|
|
76
77
|
for (const font of snapshot.library.part.fonts)
|
|
@@ -217,7 +218,13 @@ function projectSnapshot(deps, id, title) {
|
|
|
217
218
|
if (rows.length > 0)
|
|
218
219
|
tables[table] = rows;
|
|
219
220
|
}
|
|
220
|
-
|
|
221
|
+
// Files go in by their stored paths, as every version before readable folders wrote them:
|
|
222
|
+
// a moved file (`places.ts`) is named by the path its rows hold, and the import puts it
|
|
223
|
+
// there for its own arranging to move. The marker names this install's places; it stays.
|
|
224
|
+
const files = projectFiles(projectDir(deps.paths, id))
|
|
225
|
+
.filter((file) => file.path !== projectMarker)
|
|
226
|
+
.map((file) => ({ ...file, path: deps.paths.places?.storedAt(id, file.path) ?? file.path }));
|
|
227
|
+
return { title, part: { id, tables, files } };
|
|
221
228
|
}
|
|
222
229
|
// Every plain file under the project's folder, symlinks and all else left out. Not only the
|
|
223
230
|
// rows' files: audio chunks a retry would reuse and the files a revision history keeps are
|
|
@@ -31,6 +31,9 @@ export function deleteProject(deps, projectId) {
|
|
|
31
31
|
// ON DELETE CASCADE on the project, and the connection runs with foreign keys on
|
|
32
32
|
// (kernel/db/index.ts). A deleted project leaves no files and no rows.
|
|
33
33
|
deps.db.prepare("DELETE FROM projects WHERE id = ?").run(projectId);
|
|
34
|
+
const places = deps.paths.places;
|
|
35
|
+
if (places !== undefined && "forget" in places)
|
|
36
|
+
places.forget(projectId);
|
|
34
37
|
return { ok: true };
|
|
35
38
|
}
|
|
36
39
|
function messageOf(error) {
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { extname, isAbsolute, relative, resolve } from "node:path";
|
|
2
|
+
// A project's folder: its readable name once it has one (`places.ts`), its id until then.
|
|
2
3
|
export function projectDir(paths, projectId) {
|
|
3
|
-
return contained(paths.projects, projectId);
|
|
4
|
+
return contained(paths.projects, paths.places?.folderOf(projectId) ?? projectId);
|
|
4
5
|
}
|
|
5
6
|
// The clips of each project's last render (`slices/video/clip-cache.ts`). Hidden, so
|
|
6
7
|
// reconcile, backups and the user's file browser leave it alone; disposable, so moving the
|
|
@@ -9,8 +10,9 @@ export const renderCacheFolder = ".render-cache";
|
|
|
9
10
|
export function renderCacheDir(paths, projectId) {
|
|
10
11
|
return contained(paths.projects, `${renderCacheFolder}/${projectId}`);
|
|
11
12
|
}
|
|
13
|
+
// A stored path on disk: where it was moved to (`places.ts`), or the stored path itself.
|
|
12
14
|
export function outputPath(paths, projectId, relativePath) {
|
|
13
|
-
return contained(projectDir(paths, projectId), relativePath);
|
|
15
|
+
return contained(projectDir(paths, projectId), paths.places?.placeOf(projectId, relativePath) ?? relativePath);
|
|
14
16
|
}
|
|
15
17
|
// Scheduled backups (slices/backups) land in paths.backups unless the user picks another
|
|
16
18
|
// folder. Installs from before 3.0 keep it inside the projects root, since that is the one
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
import { existsSync } from "node:fs";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
export function createPlaces(db) {
|
|
4
|
+
const folders = new Map();
|
|
5
|
+
const files = new Map();
|
|
6
|
+
const stored = new Map();
|
|
7
|
+
for (const row of db.prepare("SELECT * FROM project_folders").all())
|
|
8
|
+
folders.set(String(row.project_id), {
|
|
9
|
+
folder: String(row.folder),
|
|
10
|
+
previous: row.previous === null ? undefined : String(row.previous),
|
|
11
|
+
state: row.state === "moving" ? "moving" : "placed",
|
|
12
|
+
arranged: row.arranged === null ? undefined : String(row.arranged),
|
|
13
|
+
});
|
|
14
|
+
for (const row of db.prepare("SELECT * FROM file_places").all())
|
|
15
|
+
setFile(String(row.project_id), String(row.path), {
|
|
16
|
+
place: String(row.place),
|
|
17
|
+
previous: row.previous === null ? undefined : String(row.previous),
|
|
18
|
+
state: row.state === "moving" ? "moving" : "placed",
|
|
19
|
+
});
|
|
20
|
+
function setFile(projectId, path, place) {
|
|
21
|
+
let own = files.get(projectId);
|
|
22
|
+
let back = stored.get(projectId);
|
|
23
|
+
const old = own?.get(path);
|
|
24
|
+
if (old !== undefined)
|
|
25
|
+
back?.delete(old.place.toLowerCase());
|
|
26
|
+
if (place === undefined) {
|
|
27
|
+
own?.delete(path);
|
|
28
|
+
return;
|
|
29
|
+
}
|
|
30
|
+
if (own === undefined) {
|
|
31
|
+
own = new Map();
|
|
32
|
+
files.set(projectId, own);
|
|
33
|
+
}
|
|
34
|
+
if (back === undefined) {
|
|
35
|
+
back = new Map();
|
|
36
|
+
stored.set(projectId, back);
|
|
37
|
+
}
|
|
38
|
+
own.set(path, place);
|
|
39
|
+
back.set(place.place.toLowerCase(), path);
|
|
40
|
+
}
|
|
41
|
+
const moves = {
|
|
42
|
+
folder(projectId, folder, previous) {
|
|
43
|
+
db.prepare(`INSERT INTO project_folders(project_id,folder,previous,state) VALUES (?,?,?,'moving')
|
|
44
|
+
ON CONFLICT(project_id) DO UPDATE SET folder=excluded.folder,previous=excluded.previous,state='moving'`).run(projectId, folder, previous ?? null);
|
|
45
|
+
folders.set(projectId, {
|
|
46
|
+
folder,
|
|
47
|
+
previous,
|
|
48
|
+
state: "moving",
|
|
49
|
+
arranged: folders.get(projectId)?.arranged,
|
|
50
|
+
});
|
|
51
|
+
},
|
|
52
|
+
folderDone(projectId) {
|
|
53
|
+
db.prepare("UPDATE project_folders SET state='placed',previous=NULL WHERE project_id=?").run(projectId);
|
|
54
|
+
const row = folders.get(projectId);
|
|
55
|
+
if (row !== undefined)
|
|
56
|
+
folders.set(projectId, { ...row, state: "placed", previous: undefined });
|
|
57
|
+
},
|
|
58
|
+
folderUndone(projectId) {
|
|
59
|
+
const row = folders.get(projectId);
|
|
60
|
+
if (row?.previous === undefined) {
|
|
61
|
+
db.prepare("DELETE FROM project_folders WHERE project_id=?").run(projectId);
|
|
62
|
+
folders.delete(projectId);
|
|
63
|
+
return;
|
|
64
|
+
}
|
|
65
|
+
db.prepare("UPDATE project_folders SET folder=previous,previous=NULL,state='placed' WHERE project_id=?").run(projectId);
|
|
66
|
+
folders.set(projectId, {
|
|
67
|
+
...row,
|
|
68
|
+
folder: row.previous,
|
|
69
|
+
previous: undefined,
|
|
70
|
+
state: "placed",
|
|
71
|
+
});
|
|
72
|
+
},
|
|
73
|
+
file(projectId, path, place, previous) {
|
|
74
|
+
db.prepare(`INSERT INTO file_places(project_id,path,place,previous,state) VALUES (?,?,?,?,'moving')
|
|
75
|
+
ON CONFLICT(project_id,path) DO UPDATE SET place=excluded.place,previous=excluded.previous,state='moving'`).run(projectId, path, place, previous ?? null);
|
|
76
|
+
setFile(projectId, path, { place, previous, state: "moving" });
|
|
77
|
+
},
|
|
78
|
+
fileDone(projectId, path) {
|
|
79
|
+
db.prepare("UPDATE file_places SET state='placed',previous=NULL WHERE project_id=? AND path=?").run(projectId, path);
|
|
80
|
+
const row = files.get(projectId)?.get(path);
|
|
81
|
+
if (row !== undefined)
|
|
82
|
+
setFile(projectId, path, { ...row, state: "placed", previous: undefined });
|
|
83
|
+
},
|
|
84
|
+
fileUndone(projectId, path) {
|
|
85
|
+
const row = files.get(projectId)?.get(path);
|
|
86
|
+
if (row?.previous === undefined || row.previous === path) {
|
|
87
|
+
db.prepare("DELETE FROM file_places WHERE project_id=? AND path=?").run(projectId, path);
|
|
88
|
+
setFile(projectId, path, undefined);
|
|
89
|
+
return;
|
|
90
|
+
}
|
|
91
|
+
db.prepare("UPDATE file_places SET place=previous,previous=NULL,state='placed' WHERE project_id=? AND path=?").run(projectId, path);
|
|
92
|
+
setFile(projectId, path, { place: row.previous, previous: undefined, state: "placed" });
|
|
93
|
+
},
|
|
94
|
+
arranged(projectId, key) {
|
|
95
|
+
db.prepare("UPDATE project_folders SET arranged=? WHERE project_id=?").run(key, projectId);
|
|
96
|
+
const row = folders.get(projectId);
|
|
97
|
+
if (row !== undefined)
|
|
98
|
+
folders.set(projectId, { ...row, arranged: key });
|
|
99
|
+
},
|
|
100
|
+
arrangedKey(projectId) {
|
|
101
|
+
return folders.get(projectId)?.arranged;
|
|
102
|
+
},
|
|
103
|
+
};
|
|
104
|
+
return {
|
|
105
|
+
// A move in flight still answers with where the file is until it is done.
|
|
106
|
+
folderOf(projectId) {
|
|
107
|
+
const row = folders.get(projectId);
|
|
108
|
+
if (row === undefined)
|
|
109
|
+
return undefined;
|
|
110
|
+
return row.state === "moving" ? row.previous : row.folder;
|
|
111
|
+
},
|
|
112
|
+
placeOf(projectId, path) {
|
|
113
|
+
const row = files.get(projectId)?.get(path);
|
|
114
|
+
if (row === undefined)
|
|
115
|
+
return undefined;
|
|
116
|
+
return row.state === "moving" ? row.previous : row.place;
|
|
117
|
+
},
|
|
118
|
+
storedAt(projectId, place) {
|
|
119
|
+
return stored.get(projectId)?.get(place.toLowerCase());
|
|
120
|
+
},
|
|
121
|
+
placesOf(projectId) {
|
|
122
|
+
const own = files.get(projectId);
|
|
123
|
+
const out = new Map();
|
|
124
|
+
for (const [path, row] of own ?? [])
|
|
125
|
+
if (row.state === "placed")
|
|
126
|
+
out.set(path, row.place);
|
|
127
|
+
return out;
|
|
128
|
+
},
|
|
129
|
+
forget(projectId) {
|
|
130
|
+
folders.delete(projectId);
|
|
131
|
+
files.delete(projectId);
|
|
132
|
+
stored.delete(projectId);
|
|
133
|
+
},
|
|
134
|
+
moves,
|
|
135
|
+
};
|
|
136
|
+
}
|
|
137
|
+
// Settles moves a crash interrupted, from what is on disk: a move whose target exists and
|
|
138
|
+
// whose source doesn't happened; any other is taken back. Run before anything reads a file.
|
|
139
|
+
export function recoverPlaces(db, paths) {
|
|
140
|
+
let settled = 0;
|
|
141
|
+
for (const row of db.prepare("SELECT * FROM project_folders WHERE state='moving'").all()) {
|
|
142
|
+
const id = String(row.project_id);
|
|
143
|
+
const target = join(paths.projects, String(row.folder));
|
|
144
|
+
const source = join(paths.projects, row.previous === null ? id : String(row.previous));
|
|
145
|
+
if (existsSync(target) && !existsSync(source))
|
|
146
|
+
db.prepare("UPDATE project_folders SET state='placed',previous=NULL WHERE project_id=?").run(id);
|
|
147
|
+
else if (row.previous === null)
|
|
148
|
+
db.prepare("DELETE FROM project_folders WHERE project_id=?").run(id);
|
|
149
|
+
else
|
|
150
|
+
db.prepare("UPDATE project_folders SET folder=previous,previous=NULL,state='placed' WHERE project_id=?").run(id);
|
|
151
|
+
settled += 1;
|
|
152
|
+
}
|
|
153
|
+
const folderOf = new Map(db
|
|
154
|
+
.prepare("SELECT project_id,folder FROM project_folders")
|
|
155
|
+
.all()
|
|
156
|
+
.map((row) => [String(row.project_id), String(row.folder)]));
|
|
157
|
+
for (const row of db.prepare("SELECT * FROM file_places WHERE state='moving'").all()) {
|
|
158
|
+
const id = String(row.project_id);
|
|
159
|
+
const root = join(paths.projects, folderOf.get(id) ?? id);
|
|
160
|
+
const path = String(row.path);
|
|
161
|
+
const previous = row.previous === null ? path : String(row.previous);
|
|
162
|
+
if (existsSync(join(root, String(row.place))) && !existsSync(join(root, previous)))
|
|
163
|
+
db.prepare("UPDATE file_places SET state='placed',previous=NULL WHERE project_id=? AND path=?").run(id, path);
|
|
164
|
+
else if (row.previous === null || previous === path)
|
|
165
|
+
db.prepare("DELETE FROM file_places WHERE project_id=? AND path=?").run(id, path);
|
|
166
|
+
else
|
|
167
|
+
db.prepare("UPDATE file_places SET place=previous,previous=NULL,state='placed' WHERE project_id=? AND path=?").run(id, path);
|
|
168
|
+
settled += 1;
|
|
169
|
+
}
|
|
170
|
+
return settled;
|
|
171
|
+
}
|
|
@@ -1,10 +1,17 @@
|
|
|
1
1
|
import { readdirSync, rmSync, statSync, unlinkSync } from "node:fs";
|
|
2
2
|
import { isAbsolute, join, relative, sep } from "node:path";
|
|
3
|
+
import { adoptMarked, markerProject, projectMarker } from "./arrange.js";
|
|
3
4
|
import { backupsFolderName, outputPath, stagingPath } from "./layout.js";
|
|
4
5
|
import { stagedFiles as stagedRows } from "./repo.js";
|
|
5
6
|
import { stagedFileReferenced } from "./staging-refs.js";
|
|
6
7
|
export function reconcileStorage(db, paths) {
|
|
7
8
|
const projects = idsOf(db, "SELECT id FROM projects", "id");
|
|
9
|
+
const folders = new Map();
|
|
10
|
+
for (const project of projects) {
|
|
11
|
+
const folder = paths.places?.folderOf(project);
|
|
12
|
+
if (folder !== undefined)
|
|
13
|
+
folders.set(folder.toLowerCase(), project);
|
|
14
|
+
}
|
|
8
15
|
const kept = new Set();
|
|
9
16
|
for (const row of db
|
|
10
17
|
.prepare("SELECT project_id, path FROM outputs UNION SELECT project_id, path FROM project_assets")
|
|
@@ -13,7 +20,7 @@ export function reconcileStorage(db, paths) {
|
|
|
13
20
|
const path = row.path;
|
|
14
21
|
if (typeof projectId === "string" && typeof path === "string") {
|
|
15
22
|
outputPath(paths, projectId, path);
|
|
16
|
-
kept.add(`${projectId}/${slashed(path)}`);
|
|
23
|
+
kept.add(`${projectId}/${slashed(placed(paths, projectId, path))}`);
|
|
17
24
|
}
|
|
18
25
|
}
|
|
19
26
|
// A stage's unfinished work is on the project too: the audio chunks a previous run
|
|
@@ -27,7 +34,7 @@ export function reconcileStorage(db, paths) {
|
|
|
27
34
|
const projectId = row.project_id;
|
|
28
35
|
const file = pieceFile(row.payload);
|
|
29
36
|
if (typeof projectId === "string" && file !== undefined) {
|
|
30
|
-
kept.add(`${projectId}/${slashed(file)}`);
|
|
37
|
+
kept.add(`${projectId}/${slashed(placed(paths, projectId, file))}`);
|
|
31
38
|
}
|
|
32
39
|
}
|
|
33
40
|
let orphanFiles = 0;
|
|
@@ -50,15 +57,25 @@ export function reconcileStorage(db, paths) {
|
|
|
50
57
|
orphanFiles += 1;
|
|
51
58
|
continue;
|
|
52
59
|
}
|
|
53
|
-
|
|
60
|
+
// A folder is a project's by its readable name (`places.ts`), its id, or the marker
|
|
61
|
+
// arranging left in it (`arrange.ts`). A deleted project's folder goes; a folder that is
|
|
62
|
+
// none of these is the person's own and stays.
|
|
63
|
+
const owner = folderOwner(db, paths, projects, folders, entry.name, path);
|
|
64
|
+
if (owner === "foreign")
|
|
65
|
+
continue;
|
|
66
|
+
if (owner === "deleted") {
|
|
54
67
|
orphanFiles += filesUnder(path).length;
|
|
55
68
|
rmSync(path, { recursive: true, force: true });
|
|
56
69
|
continue;
|
|
57
70
|
}
|
|
58
71
|
for (const file of filesUnder(path)) {
|
|
59
|
-
|
|
72
|
+
const name = slashed(file);
|
|
73
|
+
if (kept.has(`${owner}/${name}`))
|
|
74
|
+
continue;
|
|
75
|
+
// The marker, and anything in the folders a person browses: a note or a file dropped
|
|
76
|
+
// next to the video is theirs. Only Slopify's own leftovers elsewhere are removed.
|
|
77
|
+
if (name === projectMarker || arrangedArea.test(name))
|
|
60
78
|
continue;
|
|
61
|
-
}
|
|
62
79
|
unlinkSync(join(path, file));
|
|
63
80
|
orphanFiles += 1;
|
|
64
81
|
}
|
|
@@ -108,6 +125,38 @@ export function pieceFile(payload) {
|
|
|
108
125
|
const file = parsed.file;
|
|
109
126
|
return typeof file === "string" && file !== "" ? file : undefined;
|
|
110
127
|
}
|
|
128
|
+
// Where a stored path is on disk now, relative to its project folder.
|
|
129
|
+
function placed(paths, projectId, path) {
|
|
130
|
+
return paths.places?.placeOf(projectId, path) ?? path;
|
|
131
|
+
}
|
|
132
|
+
// The folders arranging makes (`arrange.ts`): a file in them that no record names is the
|
|
133
|
+
// person's, not a leftover.
|
|
134
|
+
const arrangedArea = /^(Upload|Working|History)\//u;
|
|
135
|
+
const ulid = /^[0-9A-HJKMNP-TV-Z]{26}$/u;
|
|
136
|
+
// Whose a folder in the projects root is: a project's id, "deleted" (a project that no longer
|
|
137
|
+
// exists: its id, or its marker's), or "foreign" (not Slopify's).
|
|
138
|
+
function folderOwner(db, paths, projects, folders, name, path) {
|
|
139
|
+
const named = folders.get(name.toLowerCase());
|
|
140
|
+
if (named !== undefined)
|
|
141
|
+
return named;
|
|
142
|
+
const marked = markerProject(path);
|
|
143
|
+
if (marked !== undefined) {
|
|
144
|
+
if (!projects.has(marked))
|
|
145
|
+
return "deleted";
|
|
146
|
+
// The database lost this folder's record (restored from before it was arranged): its
|
|
147
|
+
// marker says whose it is, and where its files went.
|
|
148
|
+
if (paths.places !== undefined && paths.places.folderOf(marked) === undefined) {
|
|
149
|
+
adoptMarked(db, paths, marked, name, path);
|
|
150
|
+
return marked;
|
|
151
|
+
}
|
|
152
|
+
return paths.places?.folderOf(marked)?.toLowerCase() === name.toLowerCase()
|
|
153
|
+
? marked
|
|
154
|
+
: "foreign";
|
|
155
|
+
}
|
|
156
|
+
if (projects.has(name))
|
|
157
|
+
return paths.places?.folderOf(name) === undefined ? name : "foreign";
|
|
158
|
+
return ulid.test(name) ? "deleted" : "foreign";
|
|
159
|
+
}
|
|
111
160
|
function idsOf(db, sql, column) {
|
|
112
161
|
const ids = new Set();
|
|
113
162
|
for (const row of db.prepare(sql).all()) {
|
|
@@ -2,7 +2,7 @@ import { existsSync, readdirSync, statSync, unlinkSync } from "node:fs";
|
|
|
2
2
|
import { join, relative, sep } from "node:path";
|
|
3
3
|
import { derive } from "../../kernel/runner/graph.js";
|
|
4
4
|
import { projectExists, projectPaused, stagesOf } from "../admission/repo.js";
|
|
5
|
-
import { backupsFolderName, projectDir } from "./layout.js";
|
|
5
|
+
import { backupsFolderName, outputPath, projectDir } from "./layout.js";
|
|
6
6
|
import { pieceFile } from "./reconcile.js";
|
|
7
7
|
// Keep outputs only: a finished project drops the working files it was made from (images,
|
|
8
8
|
// narration parts, subtitle timing, render settings) and keeps what gets published. Two
|
|
@@ -68,7 +68,7 @@ export function keepOutputsOnly(deps, projectId) {
|
|
|
68
68
|
const bytes = files.get(path);
|
|
69
69
|
if (bytes === undefined)
|
|
70
70
|
continue;
|
|
71
|
-
unlinkSync(
|
|
71
|
+
unlinkSync(outputPath(deps.paths, projectId, path));
|
|
72
72
|
count += 1;
|
|
73
73
|
bytesFreed += bytes;
|
|
74
74
|
}
|
|
@@ -85,7 +85,7 @@ function finished(deps, projectId) {
|
|
|
85
85
|
.prepare("SELECT 1 FROM revision_work WHERE project_id=? AND (state='running' OR (state='pending' AND dispatch_state='allowed')) LIMIT 1")
|
|
86
86
|
.get(projectId) === undefined);
|
|
87
87
|
}
|
|
88
|
-
// Every file under the project folder, by
|
|
88
|
+
// Every file under the project folder, by stored path with forward slashes.
|
|
89
89
|
function filesOf(paths, projectId) {
|
|
90
90
|
const files = new Map();
|
|
91
91
|
if (projectId === backupsFolderName)
|
|
@@ -97,7 +97,9 @@ function filesOf(paths, projectId) {
|
|
|
97
97
|
if (!entry.isFile())
|
|
98
98
|
continue;
|
|
99
99
|
const full = join(entry.parentPath, entry.name);
|
|
100
|
-
|
|
100
|
+
const place = relative(root, full).split(sep).join("/");
|
|
101
|
+
// Keyed by stored path: a file moved by arranging (`places.ts`) is found by where it is.
|
|
102
|
+
files.set(paths.places?.storedAt(projectId, place) ?? place, statSync(full).size);
|
|
101
103
|
}
|
|
102
104
|
return files;
|
|
103
105
|
}
|
|
@@ -25,6 +25,24 @@ The terminal prints both places when Slopify starts (`Slopify data directory:` a
|
|
|
25
25
|
|
|
26
26
|
The folder is created if it is missing.
|
|
27
27
|
|
|
28
|
+
## Inside a project folder
|
|
29
|
+
|
|
30
|
+
Each project has one folder in `Projects/`, named after its title (`Lolth | D&D Lore To Sleep To` becomes `Lolth - D&D Lore To Sleep To`). Inside it:
|
|
31
|
+
|
|
32
|
+
| Folder | What is in it | Safe to delete? |
|
|
33
|
+
| --- | --- | --- |
|
|
34
|
+
| `Upload/` | The current version, ready to publish: `video.mp4`, `thumbnail.png` (and `thumbnail 2.png`…), `subtitles.srt` and `.vtt`, `description.txt`, `tags.txt`, `titles.txt`, `pinned comment.txt`, `document.pdf`, `article.md`, and `Shorts/1 - <title>.mp4`… | No: it is what you upload. |
|
|
35
|
+
| `Working/` | What the current version was made from: `Images/`, `Narration/` (each chunk, the joined and levelled narration), `Research/`, `Article/`, `Subtitles/`, `Render/`, `Shorts/` (each short's images). | Only to free space on a finished video: changing that project later makes these again, which takes time and provider credits. **Keep outputs only** does it for you. |
|
|
36
|
+
| `History/<date>/` | Files of older versions, by the day they were made (an earlier render, the narration before a voice change). | Yes. Delete a day's folder, or all of `History/`, to free its space. Going back to that version in History makes its files again. |
|
|
37
|
+
|
|
38
|
+
A hidden `.slopify-project` file tells Slopify which project the folder holds and where its files went. Leave it there.
|
|
39
|
+
|
|
40
|
+
Slopify sorts a project's files into these folders when nothing of it is running, so a project shows them a little after it finishes (or after an edit). While a run is going, its new files sit in `assets/` until then. Renaming a project renames its folder the same way.
|
|
41
|
+
|
|
42
|
+
Files you add yourself to `Upload/`, `Working/` or `History/` (a note, a reworked thumbnail) are never removed by Slopify. A folder in `Projects/` that Slopify did not make is left alone.
|
|
43
|
+
|
|
44
|
+
**After this change, don't go back to a Slopify older than 3.14:** older versions only know folders named by project id, and remove the readable ones when they start.
|
|
45
|
+
|
|
28
46
|
## New and existing installs
|
|
29
47
|
|
|
30
48
|
Slopify decides once, the first time 3.0 starts, and remembers it:
|
|
@@ -126,7 +144,7 @@ A finished project's own page offers the same in its right rail: **Free space**
|
|
|
126
144
|
|
|
127
145
|
### Clear leftover files
|
|
128
146
|
|
|
129
|
-
**Clear leftover files** deletes files in the projects folder that no project records any more, and uploaded files that no draft, template or project uses. Your projects, outputs and library are never touched. Slopify also does this each time it starts; use it after a crash or to free space now.
|
|
147
|
+
**Clear leftover files** deletes files in the projects folder that no project records any more (never anything in a project's `Upload/`, `Working/` or `History/`, and never a folder Slopify did not make), and uploaded files that no draft, template or project uses. Your projects, outputs and library are never touched. Slopify also does this each time it starts; use it after a crash or to free space now.
|
|
130
148
|
|
|
131
149
|
Cleanup only looks inside `Projects`. It leaves `Backups` alone (only Slopify's own `slopify-backup-*.tar` files there are ever pruned, by the backup rules), leaves hidden files such as `.DS_Store` and `desktop.ini`, never removes loose files in a `Projects` folder outside the data folder, and never touches `Exports` or anything else in your files folder.
|
|
132
150
|
|