@axium/storage 0.8.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/client/api.d.ts +1 -1
- package/dist/client/api.js +9 -2
- package/dist/client/cli.js +102 -32
- package/dist/client/config.d.ts +7 -5
- package/dist/client/config.js +3 -4
- package/dist/client/hooks.js +1 -1
- package/dist/client/local.d.ts +58 -0
- package/dist/client/local.js +90 -0
- package/dist/client/sync.d.ts +21 -9
- package/dist/client/sync.js +112 -121
- package/dist/common.d.ts +61 -22
- package/dist/common.js +19 -1
- package/dist/node.d.ts +13 -0
- package/dist/node.js +52 -0
- package/dist/server/api.js +6 -6
- package/dist/server/batch.js +3 -3
- package/dist/server/db.d.ts +4 -3
- package/dist/server/db.js +9 -5
- package/dist/server/raw.js +6 -6
- package/lib/Usage.svelte +4 -4
- package/package.json +2 -2
- package/routes/files/usage/+page.svelte +3 -5
- package/dist/client/paths.d.ts +0 -2
- package/dist/client/paths.js +0 -16
package/dist/client/api.d.ts
CHANGED
|
@@ -15,7 +15,7 @@ export declare function downloadItem(fileId: string): Promise<Blob>;
|
|
|
15
15
|
export declare function updateItemMetadata(fileId: string, metadata: StorageItemUpdate): Promise<StorageItemMetadata>;
|
|
16
16
|
export declare function deleteItem(fileId: string): Promise<StorageItemMetadata>;
|
|
17
17
|
export declare function getUserStorage(userId: string): Promise<UserStorage>;
|
|
18
|
-
export declare function
|
|
18
|
+
export declare function getUserStats(userId: string): Promise<UserStorageInfo>;
|
|
19
19
|
export declare function getUserTrash(userId: string): Promise<StorageItemMetadata[]>;
|
|
20
20
|
export declare function itemsSharedWith(userId: string): Promise<StorageItemMetadata[]>;
|
|
21
21
|
export declare function getUserStorageRoot(userId: string): Promise<StorageItemMetadata[]>;
|
package/dist/client/api.js
CHANGED
|
@@ -80,12 +80,19 @@ export async function deleteItem(fileId) {
|
|
|
80
80
|
}
|
|
81
81
|
export async function getUserStorage(userId) {
|
|
82
82
|
const result = await fetchAPI('GET', 'users/:id/storage', undefined, userId);
|
|
83
|
+
result.lastModified = new Date(result.lastModified);
|
|
84
|
+
if (result.lastTrashed)
|
|
85
|
+
result.lastTrashed = new Date(result.lastTrashed);
|
|
83
86
|
for (const item of result.items)
|
|
84
87
|
parseItem(item);
|
|
85
88
|
return result;
|
|
86
89
|
}
|
|
87
|
-
export async function
|
|
88
|
-
|
|
90
|
+
export async function getUserStats(userId) {
|
|
91
|
+
const result = await fetchAPI('OPTIONS', 'users/:id/storage', undefined, userId);
|
|
92
|
+
result.lastModified = new Date(result.lastModified);
|
|
93
|
+
if (result.lastTrashed)
|
|
94
|
+
result.lastTrashed = new Date(result.lastTrashed);
|
|
95
|
+
return result;
|
|
89
96
|
}
|
|
90
97
|
export async function getUserTrash(userId) {
|
|
91
98
|
const result = await fetchAPI('GET', 'users/:id/storage/trash', undefined, userId);
|
package/dist/client/cli.js
CHANGED
|
@@ -6,23 +6,61 @@ import { statSync, unlinkSync } from 'node:fs';
|
|
|
6
6
|
import { stat } from 'node:fs/promises';
|
|
7
7
|
import { join, resolve } from 'node:path';
|
|
8
8
|
import { styleText } from 'node:util';
|
|
9
|
-
import {
|
|
9
|
+
import { colorItem, formatItems } from '../node.js';
|
|
10
|
+
import * as api from './api.js';
|
|
10
11
|
import { config, saveConfig } from './config.js';
|
|
11
|
-
import {
|
|
12
|
+
import { cachePath, getDirectory, resolveItem, setQuiet, syncCache } from './local.js';
|
|
12
13
|
import { computeDelta, doSync, fetchSyncItems } from './sync.js';
|
|
13
|
-
const cli = program
|
|
14
|
+
const cli = program
|
|
15
|
+
.command('files')
|
|
16
|
+
.helpGroup('Plugins:')
|
|
17
|
+
.description('CLI integration for @axium/storage')
|
|
18
|
+
.option('-q, --quiet', 'Suppress output')
|
|
19
|
+
.hook('preAction', (action) => {
|
|
20
|
+
const opts = action.optsWithGlobals();
|
|
21
|
+
if (opts.quiet)
|
|
22
|
+
setQuiet(true);
|
|
23
|
+
});
|
|
14
24
|
cli.command('usage')
|
|
15
25
|
.description('Show your usage')
|
|
16
26
|
.action(async () => {
|
|
17
|
-
const { limits,
|
|
18
|
-
console.log(`Items: ${
|
|
19
|
-
console.log(`Space: ${formatBytes(
|
|
27
|
+
const { limits, itemCount, usedBytes } = await api.getUserStats(session().userId);
|
|
28
|
+
console.log(`Items: ${itemCount} ${limits.user_items ? ' / ' + limits.user_items : ''}`);
|
|
29
|
+
console.log(`Space: ${formatBytes(usedBytes)} ${limits.user_size ? ' / ' + formatBytes(limits.user_size * 1_000_000) : ''}`);
|
|
20
30
|
});
|
|
21
31
|
cli.command('ls')
|
|
22
32
|
.alias('list')
|
|
23
33
|
.description('List the contents of a folder')
|
|
24
|
-
.
|
|
25
|
-
|
|
34
|
+
.argument('<path>', 'remote folder path')
|
|
35
|
+
.option('-l, --long', 'Show more details')
|
|
36
|
+
.option('-h, --human-readable', 'Show sizes in human readable format')
|
|
37
|
+
.action(async function (path) {
|
|
38
|
+
const { users } = await syncCache().catch(io.handleError);
|
|
39
|
+
const { long, humanReadable } = this.optsWithGlobals();
|
|
40
|
+
const items = await getDirectory(path).catch(io.handleError);
|
|
41
|
+
if (!long) {
|
|
42
|
+
console.log(items.map(colorItem).join('\t'));
|
|
43
|
+
return;
|
|
44
|
+
}
|
|
45
|
+
console.log('total ' + items.length);
|
|
46
|
+
for (const text of formatItems({ items, users, humanReadable }))
|
|
47
|
+
console.log(text);
|
|
48
|
+
});
|
|
49
|
+
cli.command('mkdir')
|
|
50
|
+
.description('Create a remote folder')
|
|
51
|
+
.argument('<path>', 'remote folder path to create')
|
|
52
|
+
.action(async (path) => {
|
|
53
|
+
const pathParts = path.split('/');
|
|
54
|
+
const name = pathParts.pop();
|
|
55
|
+
const parentPath = pathParts.join('/');
|
|
56
|
+
const parent = !parentPath ? null : await resolveItem(parentPath).catch(io.handleError);
|
|
57
|
+
if (parent) {
|
|
58
|
+
if (!parent)
|
|
59
|
+
io.exit('Could not resolve parent folder.');
|
|
60
|
+
if (parent.type != 'inode/directory')
|
|
61
|
+
io.exit('Parent path is not a directory.');
|
|
62
|
+
}
|
|
63
|
+
await api.uploadItem(new Blob([], { type: 'inode/directory' }), { parentId: parent?.id, name }).catch(io.handleError);
|
|
26
64
|
});
|
|
27
65
|
cli.command('status')
|
|
28
66
|
.option('-v, --verbose', 'Show more details')
|
|
@@ -31,26 +69,26 @@ cli.command('status')
|
|
|
31
69
|
console.log(styleText('bold', `${config.sync.length} synced folder(s):`));
|
|
32
70
|
for (const sync of config.sync) {
|
|
33
71
|
if (opt.refresh)
|
|
34
|
-
await fetchSyncItems(sync.
|
|
35
|
-
const delta = computeDelta(sync
|
|
72
|
+
await fetchSyncItems(sync.item, sync.name);
|
|
73
|
+
const delta = computeDelta(sync);
|
|
36
74
|
if (opt.verbose) {
|
|
37
|
-
console.log(styleText('underline', sync.
|
|
38
|
-
if (delta.synced.length == delta.items.length && !delta.
|
|
75
|
+
console.log(styleText('underline', sync.local_path + ':'));
|
|
76
|
+
if (delta.synced.length == delta.items.length && !delta.local_only.length) {
|
|
39
77
|
console.log('\t' + styleText('blueBright', 'All files are synced!'));
|
|
40
78
|
continue;
|
|
41
79
|
}
|
|
42
|
-
for (const { _path } of delta.
|
|
80
|
+
for (const { _path } of delta.local_only)
|
|
43
81
|
console.log('\t' + styleText('green', '+ ' + _path));
|
|
44
|
-
for (const { path } of delta.
|
|
82
|
+
for (const { path } of delta.remote_only)
|
|
45
83
|
console.log('\t' + styleText('red', '- ' + path));
|
|
46
84
|
for (const { path, modifiedAt } of delta.modified) {
|
|
47
|
-
const outdated = modifiedAt.getTime() > statSync(join(sync.
|
|
85
|
+
const outdated = modifiedAt.getTime() > statSync(join(sync.local_path, path)).mtime.getTime();
|
|
48
86
|
console.log('\t' + styleText('yellow', '~ ' + path) + (outdated ? ' (outdated)' : ''));
|
|
49
87
|
}
|
|
50
88
|
}
|
|
51
89
|
else {
|
|
52
|
-
console.log(sync.
|
|
53
|
-
.map(([name, items]) => `${styleText('blueBright', items.length.toString())} ${name}`)
|
|
90
|
+
console.log(sync.local_path + ':', Object.entries(delta)
|
|
91
|
+
.map(([name, items]) => `${styleText('blueBright', items.length.toString())} ${name.replaceAll('_', '-')}`)
|
|
54
92
|
.join(', '));
|
|
55
93
|
}
|
|
56
94
|
}
|
|
@@ -62,30 +100,27 @@ cli.command('add')
|
|
|
62
100
|
.action(async (localPath, remoteName) => {
|
|
63
101
|
localPath = resolve(localPath);
|
|
64
102
|
for (const sync of config.sync) {
|
|
65
|
-
if (sync.
|
|
103
|
+
if (sync.local_path == localPath || localPath.startsWith(sync.local_path + '/'))
|
|
66
104
|
io.exit('This local path is already being synced.');
|
|
67
|
-
if (sync.
|
|
105
|
+
if (sync.remote_path == remoteName || remoteName.startsWith(sync.remote_path + '/'))
|
|
68
106
|
io.exit('This remote path is already being synced.');
|
|
69
107
|
}
|
|
70
|
-
const { userId } = session();
|
|
71
108
|
const local = await stat(localPath).catch(e => io.exit(e.toString()));
|
|
72
109
|
if (!local.isDirectory())
|
|
73
110
|
io.exit('Local path is not a directory.');
|
|
74
|
-
|
|
75
|
-
* @todo Add an endpoint to fetch directories (maybe with the full paths?)
|
|
76
|
-
*/
|
|
77
|
-
const allItems = remoteName.includes('/') ? (await getUserStorage(userId)).items : await getUserStorageRoot(userId);
|
|
78
|
-
const remote = walkItems(remoteName, allItems);
|
|
111
|
+
const remote = await resolveItem(remoteName);
|
|
79
112
|
if (!remote)
|
|
80
113
|
io.exit('Could not resolve remote path.');
|
|
81
114
|
if (remote.type != 'inode/directory')
|
|
82
115
|
io.exit('Remote path is not a directory.');
|
|
83
116
|
config.sync.push({
|
|
84
117
|
name: remote.name,
|
|
85
|
-
|
|
86
|
-
localPath,
|
|
87
|
-
|
|
88
|
-
|
|
118
|
+
item: remote.id,
|
|
119
|
+
local_path: localPath,
|
|
120
|
+
last_synced: new Date(),
|
|
121
|
+
remote_path: remoteName,
|
|
122
|
+
include_dotfiles: false,
|
|
123
|
+
exclude: [],
|
|
89
124
|
});
|
|
90
125
|
await fetchSyncItems(remote.id);
|
|
91
126
|
saveConfig();
|
|
@@ -97,19 +132,20 @@ cli.command('unsync')
|
|
|
97
132
|
.argument('<path>', 'local path to the folder to stop syncing')
|
|
98
133
|
.action((localPath) => {
|
|
99
134
|
localPath = resolve(localPath);
|
|
100
|
-
const index = config.sync.findIndex(sync => sync.
|
|
135
|
+
const index = config.sync.findIndex(sync => sync.local_path == localPath);
|
|
101
136
|
if (index == -1)
|
|
102
137
|
io.exit('This local path is not being synced.');
|
|
103
|
-
unlinkSync(join(configDir, 'sync', config.sync[index].
|
|
138
|
+
unlinkSync(join(configDir, 'sync', config.sync[index].item + '.json'));
|
|
104
139
|
config.sync.splice(index, 1);
|
|
105
140
|
saveConfig();
|
|
106
141
|
});
|
|
107
142
|
cli.command('sync')
|
|
108
143
|
.description('Sync files')
|
|
109
|
-
.addOption(new Option('--delete', 'Delete local/remote files that were deleted remotely/locally')
|
|
144
|
+
.addOption(new Option('--delete <mode>', 'Delete local/remote files that were deleted remotely/locally')
|
|
110
145
|
.choices(['local', 'remote', 'none'])
|
|
111
146
|
.default('none'))
|
|
112
147
|
.option('-d, --dry-run', 'Show what would be done, but do not make any changes')
|
|
148
|
+
.option('-v, --verbose', 'Show more details')
|
|
113
149
|
.argument('[sync]', 'The name of the Sync to sync')
|
|
114
150
|
.action(async (name, opt) => {
|
|
115
151
|
if (name) {
|
|
@@ -122,3 +158,37 @@ cli.command('sync')
|
|
|
122
158
|
for (const sync of config.sync)
|
|
123
159
|
await doSync(sync, opt);
|
|
124
160
|
});
|
|
161
|
+
const cliCache = cli.command('cache').description('Manage the local cache');
|
|
162
|
+
cliCache
|
|
163
|
+
.command('clear')
|
|
164
|
+
.description('Clear the local cache')
|
|
165
|
+
.action(() => unlinkSync(cachePath));
|
|
166
|
+
cliCache
|
|
167
|
+
.command('refresh')
|
|
168
|
+
.description('Force a refresh of the local cache from the server')
|
|
169
|
+
.action(async () => {
|
|
170
|
+
await syncCache(true).catch(io.handleError);
|
|
171
|
+
});
|
|
172
|
+
cliCache
|
|
173
|
+
.command('dump')
|
|
174
|
+
.description('Dump the local cache')
|
|
175
|
+
.option('-v, --verbose', 'Show more details')
|
|
176
|
+
.addOption(new Option('-j, --json', 'Output as JSON').conflicts(['verbose', 'quiet']))
|
|
177
|
+
.action(async function () {
|
|
178
|
+
const opt = this.optsWithGlobals();
|
|
179
|
+
const data = await syncCache(false).catch(io.handleError);
|
|
180
|
+
if (opt.json) {
|
|
181
|
+
console.log(JSON.stringify(data));
|
|
182
|
+
return;
|
|
183
|
+
}
|
|
184
|
+
console.log(`Cache contains ${data.items.length} items and ${Object.keys(data.users).length} users.`);
|
|
185
|
+
if (opt.quiet || !opt.verbose)
|
|
186
|
+
return;
|
|
187
|
+
console.log(styleText('bold', 'Items:'));
|
|
188
|
+
for (const text of formatItems({ ...data, humanReadable: true }))
|
|
189
|
+
console.log(text);
|
|
190
|
+
console.log(styleText('bold', 'Users:'));
|
|
191
|
+
for (const user of Object.values(data.users)) {
|
|
192
|
+
console.log(user.name, `<${user.email}>`, styleText('dim', `(${user.id})`));
|
|
193
|
+
}
|
|
194
|
+
});
|
package/dist/client/config.d.ts
CHANGED
|
@@ -2,11 +2,13 @@ import * as z from 'zod';
|
|
|
2
2
|
declare const ClientStorageConfig: z.ZodObject<{
|
|
3
3
|
sync: z.ZodArray<z.ZodObject<{
|
|
4
4
|
name: z.ZodString;
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
5
|
+
item: z.ZodUUID;
|
|
6
|
+
local_path: z.ZodString;
|
|
7
|
+
last_synced: z.ZodCoercedDate<unknown>;
|
|
8
|
+
remote_path: z.ZodString;
|
|
9
|
+
include_dotfiles: z.ZodDefault<z.ZodBoolean>;
|
|
10
|
+
exclude: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
11
|
+
}, z.core.$loose>>;
|
|
10
12
|
}, z.core.$loose>;
|
|
11
13
|
export interface ClientStorageConfig extends z.infer<typeof ClientStorageConfig> {
|
|
12
14
|
}
|
package/dist/client/config.js
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { configDir } from '@axium/client/cli/config';
|
|
2
|
-
import { debug, writeJSON } from '@axium/core/node/io';
|
|
3
|
-
import { readFileSync } from 'node:fs';
|
|
2
|
+
import { debug, readJSON, writeJSON } from '@axium/core/node/io';
|
|
4
3
|
import { join } from 'node:path/posix';
|
|
5
4
|
import * as z from 'zod';
|
|
6
5
|
import { Sync } from './sync.js';
|
|
@@ -10,10 +9,10 @@ const ClientStorageConfig = z.looseObject({
|
|
|
10
9
|
const configPath = join(configDir, 'storage.json');
|
|
11
10
|
export let config = { sync: [] };
|
|
12
11
|
try {
|
|
13
|
-
config =
|
|
12
|
+
config = readJSON(configPath, ClientStorageConfig);
|
|
14
13
|
}
|
|
15
14
|
catch (e) {
|
|
16
|
-
debug('Could not load @axium/storage config: ' +
|
|
15
|
+
debug('Could not load @axium/storage config: ' + e);
|
|
17
16
|
}
|
|
18
17
|
export function saveConfig() {
|
|
19
18
|
writeJSON(configPath, config);
|
package/dist/client/hooks.js
CHANGED
|
@@ -2,5 +2,5 @@ import { config } from './config.js';
|
|
|
2
2
|
import { doSync } from './sync.js';
|
|
3
3
|
export async function run() {
|
|
4
4
|
for (const sync of config.sync)
|
|
5
|
-
await doSync(sync, { delete: 'none', dryRun: false });
|
|
5
|
+
await doSync(sync, { delete: 'none', dryRun: false, verbose: true });
|
|
6
6
|
}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import { UserPublic } from '@axium/core';
|
|
2
|
+
import * as z from 'zod';
|
|
3
|
+
import { StorageItemMetadata } from '../common.js';
|
|
4
|
+
export declare function setQuiet(enabled: boolean): void;
|
|
5
|
+
export declare function walkItems(path: string, items: StorageItemMetadata[]): StorageItemMetadata | null;
|
|
6
|
+
declare const StorageCache: z.ZodObject<{
|
|
7
|
+
items: z.ZodArray<z.ZodObject<{
|
|
8
|
+
createdAt: z.ZodCoercedDate<unknown>;
|
|
9
|
+
dataURL: z.ZodString;
|
|
10
|
+
hash: z.ZodOptional<z.ZodNullable<z.ZodString>>;
|
|
11
|
+
id: z.ZodUUID;
|
|
12
|
+
immutable: z.ZodBoolean;
|
|
13
|
+
modifiedAt: z.ZodCoercedDate<unknown>;
|
|
14
|
+
name: z.ZodString;
|
|
15
|
+
userId: z.ZodUUID;
|
|
16
|
+
parentId: z.ZodNullable<z.ZodUUID>;
|
|
17
|
+
publicPermission: z.ZodEnum<{
|
|
18
|
+
readonly None: 0;
|
|
19
|
+
readonly Read: 1;
|
|
20
|
+
readonly Comment: 2;
|
|
21
|
+
readonly Edit: 3;
|
|
22
|
+
readonly Manage: 5;
|
|
23
|
+
}> & {
|
|
24
|
+
readonly None: 0;
|
|
25
|
+
readonly Read: 1;
|
|
26
|
+
readonly Comment: 2;
|
|
27
|
+
readonly Edit: 3;
|
|
28
|
+
readonly Manage: 5;
|
|
29
|
+
};
|
|
30
|
+
size: z.ZodInt;
|
|
31
|
+
trashedAt: z.ZodNullable<z.ZodCoercedDate<unknown>>;
|
|
32
|
+
type: z.ZodString;
|
|
33
|
+
metadata: z.ZodRecord<z.ZodString, z.ZodUnknown>;
|
|
34
|
+
}, z.core.$strip>>;
|
|
35
|
+
users: z.ZodRecord<z.ZodUUID, z.ZodObject<{
|
|
36
|
+
id: z.ZodUUID;
|
|
37
|
+
name: z.ZodString;
|
|
38
|
+
email: z.ZodOptional<z.ZodEmail>;
|
|
39
|
+
emailVerified: z.ZodOptional<z.ZodOptional<z.ZodNullable<z.ZodDate>>>;
|
|
40
|
+
image: z.ZodOptional<z.ZodNullable<z.ZodURL>>;
|
|
41
|
+
preferences: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodAny>>;
|
|
42
|
+
roles: z.ZodArray<z.ZodString>;
|
|
43
|
+
registeredAt: z.ZodCoercedDate<unknown>;
|
|
44
|
+
isAdmin: z.ZodOptional<z.ZodBoolean>;
|
|
45
|
+
}, z.core.$strip>>;
|
|
46
|
+
}, z.core.$strip>;
|
|
47
|
+
interface StorageCache extends z.infer<typeof StorageCache> {
|
|
48
|
+
users: Record<string, UserPublic>;
|
|
49
|
+
}
|
|
50
|
+
export declare const cachePath: string;
|
|
51
|
+
export declare let cachedData: StorageCache | null;
|
|
52
|
+
/**
|
|
53
|
+
* @param force true => always refresh, false => never refresh, null (default) => refresh if outdated
|
|
54
|
+
*/
|
|
55
|
+
export declare function syncCache(force?: boolean | null): Promise<StorageCache>;
|
|
56
|
+
export declare function resolveItem(path: string): Promise<StorageItemMetadata | null>;
|
|
57
|
+
export declare function getDirectory(path: string): Promise<StorageItemMetadata[]>;
|
|
58
|
+
export {};
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import { cacheDir, session } from '@axium/client/cli/config';
|
|
2
|
+
import { userInfo } from '@axium/client/user';
|
|
3
|
+
import { UserPublic } from '@axium/core';
|
|
4
|
+
import * as io from '@axium/core/node/io';
|
|
5
|
+
import { ENOENT, ENOTDIR } from 'node:constants';
|
|
6
|
+
import { stat } from 'node:fs/promises';
|
|
7
|
+
import { join } from 'node:path';
|
|
8
|
+
import * as z from 'zod';
|
|
9
|
+
import { StorageItemMetadata } from '../common.js';
|
|
10
|
+
import { getUserStats, getUserStorage } from './api.js';
|
|
11
|
+
let quiet = false;
|
|
12
|
+
export function setQuiet(enabled) {
|
|
13
|
+
quiet = enabled;
|
|
14
|
+
}
|
|
15
|
+
export function walkItems(path, items) {
|
|
16
|
+
const resolved = [];
|
|
17
|
+
let currentItem = null, currentParentId = null;
|
|
18
|
+
const target = path.split('/').filter(p => p);
|
|
19
|
+
for (const part of target) {
|
|
20
|
+
for (const item of items) {
|
|
21
|
+
if (item.parentId != currentParentId || item.name != part)
|
|
22
|
+
continue;
|
|
23
|
+
currentItem = item;
|
|
24
|
+
currentParentId = item.id;
|
|
25
|
+
resolved.push(part);
|
|
26
|
+
break;
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
return currentItem;
|
|
30
|
+
}
|
|
31
|
+
const StorageCache = z.object({
|
|
32
|
+
items: StorageItemMetadata.array(),
|
|
33
|
+
users: z.record(z.uuid(), UserPublic),
|
|
34
|
+
});
|
|
35
|
+
export const cachePath = join(cacheDir, 'storage.json');
|
|
36
|
+
export let cachedData = null;
|
|
37
|
+
/**
|
|
38
|
+
* @param force true => always refresh, false => never refresh, null (default) => refresh if outdated
|
|
39
|
+
*/
|
|
40
|
+
export async function syncCache(force = null) {
|
|
41
|
+
if (cachedData)
|
|
42
|
+
return cachedData;
|
|
43
|
+
const cacheUpdated = await stat(cachePath)
|
|
44
|
+
.then(stats => stats.mtimeMs)
|
|
45
|
+
.catch(() => 0);
|
|
46
|
+
const { userId } = session();
|
|
47
|
+
let { lastModified, lastTrashed } = await getUserStats(userId);
|
|
48
|
+
lastTrashed ??= new Date(0);
|
|
49
|
+
if ((force || cacheUpdated < lastModified.getTime() || cacheUpdated < lastTrashed.getTime()) && force !== false) {
|
|
50
|
+
if (!quiet)
|
|
51
|
+
io.start('Updating and loading item metadata');
|
|
52
|
+
try {
|
|
53
|
+
const { items } = await getUserStorage(userId);
|
|
54
|
+
const users = {};
|
|
55
|
+
for (const item of items) {
|
|
56
|
+
users[item.userId] ||= await userInfo(item.userId);
|
|
57
|
+
}
|
|
58
|
+
cachedData = { items, users };
|
|
59
|
+
io.writeJSON(cachePath, cachedData);
|
|
60
|
+
if (!quiet)
|
|
61
|
+
io.done();
|
|
62
|
+
}
|
|
63
|
+
catch (e) {
|
|
64
|
+
const message = e instanceof Error ? e.message : String(e);
|
|
65
|
+
if (quiet)
|
|
66
|
+
io.exit('Failed to update item metadata: ' + message);
|
|
67
|
+
else
|
|
68
|
+
io.exit(message);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
else {
|
|
72
|
+
cachedData = io.readJSON(cachePath, StorageCache);
|
|
73
|
+
}
|
|
74
|
+
return cachedData;
|
|
75
|
+
}
|
|
76
|
+
export async function resolveItem(path) {
|
|
77
|
+
const { items } = await syncCache();
|
|
78
|
+
return walkItems(path, items);
|
|
79
|
+
}
|
|
80
|
+
export async function getDirectory(path) {
|
|
81
|
+
const { items } = await syncCache();
|
|
82
|
+
if (path == '/' || path == '')
|
|
83
|
+
return items.filter(item => item.parentId === null);
|
|
84
|
+
const dir = walkItems(path, items);
|
|
85
|
+
if (!dir)
|
|
86
|
+
throw ENOENT;
|
|
87
|
+
if (dir.type != 'inode/directory')
|
|
88
|
+
throw ENOTDIR;
|
|
89
|
+
return items.filter(item => item.parentId === dir.id);
|
|
90
|
+
}
|
package/dist/client/sync.d.ts
CHANGED
|
@@ -11,11 +11,13 @@ import '../polyfills.js';
|
|
|
11
11
|
*/
|
|
12
12
|
export declare const Sync: z.ZodObject<{
|
|
13
13
|
name: z.ZodString;
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
14
|
+
item: z.ZodUUID;
|
|
15
|
+
local_path: z.ZodString;
|
|
16
|
+
last_synced: z.ZodCoercedDate<unknown>;
|
|
17
|
+
remote_path: z.ZodString;
|
|
18
|
+
include_dotfiles: z.ZodDefault<z.ZodBoolean>;
|
|
19
|
+
exclude: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
20
|
+
}, z.core.$loose>;
|
|
19
21
|
export interface Sync extends z.infer<typeof Sync> {
|
|
20
22
|
}
|
|
21
23
|
/** Local metadata about a storage item to sync */
|
|
@@ -37,19 +39,29 @@ export interface ItemMetadata {
|
|
|
37
39
|
export interface Delta {
|
|
38
40
|
synced: LocalItem[];
|
|
39
41
|
modified: LocalItem[];
|
|
40
|
-
|
|
42
|
+
remote_only: LocalItem[];
|
|
41
43
|
items: LocalItem[];
|
|
42
44
|
_items: Map<string, LocalItem>;
|
|
43
|
-
|
|
45
|
+
local_only: (fs.Dirent<string> & {
|
|
44
46
|
_path: string;
|
|
45
47
|
})[];
|
|
46
48
|
}
|
|
47
49
|
/**
|
|
48
50
|
* Computes the changes between the local and remote, in the direction of the remote.
|
|
49
51
|
*/
|
|
50
|
-
export declare function computeDelta(
|
|
52
|
+
export declare function computeDelta(sync: Sync): Delta;
|
|
53
|
+
/**
|
|
54
|
+
* A helper to run an action on an array, with support for dry-runs and displaying a progress counter or listing each item.
|
|
55
|
+
*/
|
|
56
|
+
export declare function applyAction<T>(opt: {
|
|
57
|
+
dryRun: boolean;
|
|
58
|
+
verbose: boolean;
|
|
59
|
+
}, items: T[], label: (item: T) => string, action: (item: T) => Promise<void | string> | void | string): Promise<void>;
|
|
51
60
|
export interface SyncOptions {
|
|
52
61
|
delete: 'local' | 'remote' | 'none';
|
|
53
62
|
dryRun: boolean;
|
|
63
|
+
verbose: boolean;
|
|
64
|
+
}
|
|
65
|
+
export interface SyncStats {
|
|
54
66
|
}
|
|
55
|
-
export declare function doSync(sync: Sync, opt: SyncOptions): Promise<
|
|
67
|
+
export declare function doSync(sync: Sync, opt: SyncOptions): Promise<SyncStats>;
|
package/dist/client/sync.js
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
import { configDir, saveConfig } from '@axium/client/cli/config';
|
|
2
2
|
import { fetchAPI } from '@axium/client/requests';
|
|
3
3
|
import * as io from '@axium/core/node/io';
|
|
4
|
+
import mime from 'mime';
|
|
4
5
|
import { createHash } from 'node:crypto';
|
|
5
6
|
import * as fs from 'node:fs';
|
|
6
|
-
import { dirname, join, relative } from 'node:path';
|
|
7
|
+
import { basename, dirname, join, matchesGlob, relative } from 'node:path';
|
|
7
8
|
import { pick } from 'utilium';
|
|
8
9
|
import * as z from 'zod';
|
|
9
10
|
import '../polyfills.js';
|
|
10
11
|
import { deleteItem, downloadItem, updateItem, uploadItem } from './api.js';
|
|
11
|
-
import mime from 'mime';
|
|
12
12
|
/**
|
|
13
13
|
* A Sync is a storage item that has been selected for synchronization by the user.
|
|
14
14
|
* Importantly, it is *only* the "top-level" item.
|
|
@@ -17,12 +17,14 @@ import mime from 'mime';
|
|
|
17
17
|
* `Sync`s are client-side in nature, the server does not care about them.
|
|
18
18
|
* (we could do something fancy server-side at a later date)
|
|
19
19
|
*/
|
|
20
|
-
export const Sync = z.
|
|
20
|
+
export const Sync = z.looseObject({
|
|
21
21
|
name: z.string(),
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
22
|
+
item: z.uuid(),
|
|
23
|
+
local_path: z.string(),
|
|
24
|
+
last_synced: z.coerce.date(),
|
|
25
|
+
remote_path: z.string(),
|
|
26
|
+
include_dotfiles: z.boolean().default(false),
|
|
27
|
+
exclude: z.string().array().default([]),
|
|
26
28
|
});
|
|
27
29
|
/** Local metadata about a storage item to sync */
|
|
28
30
|
export const LocalItem = z.object({
|
|
@@ -46,11 +48,7 @@ export async function fetchSyncItems(id, folderName) {
|
|
|
46
48
|
}
|
|
47
49
|
}
|
|
48
50
|
export function getItems(id) {
|
|
49
|
-
|
|
50
|
-
const { error, data } = LocalItem.array().safeParse(items);
|
|
51
|
-
if (error)
|
|
52
|
-
throw z.prettifyError(error);
|
|
53
|
-
return data;
|
|
51
|
+
return io.readJSON(join(configDir, 'sync', id + '.json'), LocalItem.array());
|
|
54
52
|
}
|
|
55
53
|
export function setItems(id, items) {
|
|
56
54
|
fs.mkdirSync(join(configDir, 'sync'), { recursive: true });
|
|
@@ -59,16 +57,19 @@ export function setItems(id, items) {
|
|
|
59
57
|
/**
|
|
60
58
|
* Computes the changes between the local and remote, in the direction of the remote.
|
|
61
59
|
*/
|
|
62
|
-
export function computeDelta(
|
|
63
|
-
const items = new Map(getItems(
|
|
60
|
+
export function computeDelta(sync) {
|
|
61
|
+
const items = new Map(getItems(sync.item).map(i => [i.path, i]));
|
|
64
62
|
const itemsSet = new Set(items.keys());
|
|
65
|
-
const files = new Map(fs
|
|
66
|
-
|
|
63
|
+
const files = new Map(fs
|
|
64
|
+
.readdirSync(sync.local_path, { recursive: true, encoding: 'utf8', withFileTypes: true })
|
|
65
|
+
.map(d => {
|
|
66
|
+
const _path = relative(sync.local_path, join(d.parentPath, d.name));
|
|
67
67
|
return [_path, Object.assign(d, { _path })];
|
|
68
|
-
})
|
|
68
|
+
})
|
|
69
|
+
.filter(([p]) => (sync.include_dotfiles || basename(p)[0] != '.') && !sync.exclude.some(glob => matchesGlob(p, glob))));
|
|
69
70
|
const synced = itemsSet.intersection(files);
|
|
70
71
|
const modified = new Set(synced.keys().filter(path => {
|
|
71
|
-
const full = join(
|
|
72
|
+
const full = join(sync.local_path, path);
|
|
72
73
|
const hash = fs.statSync(full).isDirectory() ? null : createHash('BLAKE2b512').update(fs.readFileSync(full)).digest().toHex();
|
|
73
74
|
return hash != items.get(path)?.hash;
|
|
74
75
|
}));
|
|
@@ -77,129 +78,119 @@ export function computeDelta(id, localPath) {
|
|
|
77
78
|
items: Array.from(items.values()),
|
|
78
79
|
synced: Array.from(synced.difference(modified)).map(p => items.get(p)),
|
|
79
80
|
modified: Array.from(modified).map(p => items.get(p)),
|
|
80
|
-
|
|
81
|
-
|
|
81
|
+
local_only: Array.from(new Set(files.keys()).difference(items)).map(p => files.get(p)),
|
|
82
|
+
remote_only: Array.from(itemsSet.difference(files)).map(p => items.get(p)),
|
|
82
83
|
};
|
|
83
84
|
Object.defineProperty(delta, '_items', { enumerable: false });
|
|
84
85
|
return delta;
|
|
85
86
|
}
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
if (opt.dryRun)
|
|
87
|
+
/**
|
|
88
|
+
* A helper to run an action on an array, with support for dry-runs and displaying a progress counter or listing each item.
|
|
89
|
+
*/
|
|
90
|
+
export async function applyAction(opt, items, label, action) {
|
|
91
|
+
for (const [i, item] of items.entries()) {
|
|
92
|
+
opt.verbose ? io.start(label(item)) : io.progress(i, items.length, label(item));
|
|
93
|
+
try {
|
|
94
|
+
let result = undefined;
|
|
95
|
+
if (opt.dryRun) {
|
|
96
|
+
if (opt.verbose)
|
|
97
97
|
process.stdout.write('(dry run) ');
|
|
98
|
-
else {
|
|
99
|
-
fs.unlinkSync(join(sync.localPath, dirent._path));
|
|
100
|
-
_items.delete(dirent._path);
|
|
101
|
-
}
|
|
102
|
-
io.done();
|
|
103
98
|
}
|
|
104
|
-
|
|
105
|
-
|
|
99
|
+
else {
|
|
100
|
+
result = await action(item);
|
|
106
101
|
}
|
|
107
|
-
|
|
102
|
+
if (opt.verbose)
|
|
103
|
+
result ? io.log(result) : io.done();
|
|
104
|
+
}
|
|
105
|
+
catch (e) {
|
|
106
|
+
io.error(e.message);
|
|
107
|
+
throw e;
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
export async function doSync(sync, opt) {
|
|
112
|
+
const stats = {};
|
|
113
|
+
await fetchSyncItems(sync.item, sync.name);
|
|
114
|
+
const delta = computeDelta(sync);
|
|
115
|
+
const { _items } = delta;
|
|
116
|
+
if (!delta.local_only.length) {
|
|
117
|
+
// Nothing
|
|
118
|
+
}
|
|
119
|
+
else if (opt.delete == 'local') {
|
|
120
|
+
delta.local_only.reverse(); // so directories come after
|
|
121
|
+
if (!opt.verbose)
|
|
122
|
+
io.start('Deleting local items');
|
|
123
|
+
}
|
|
124
|
+
else if (!opt.verbose)
|
|
125
|
+
io.start('Uploading local items');
|
|
126
|
+
await applyAction(opt, delta.local_only, dirent => (!opt.verbose ? '' : opt.delete == 'local' ? 'Deleting local ' : 'Uploading ') + dirent._path, async (dirent) => {
|
|
127
|
+
if (opt.delete == 'local') {
|
|
128
|
+
fs.unlinkSync(join(sync.local_path, dirent._path));
|
|
129
|
+
_items.delete(dirent._path);
|
|
130
|
+
return;
|
|
108
131
|
}
|
|
109
132
|
const uploadOpts = {
|
|
110
|
-
parentId: dirname(dirent._path) == '.' ? sync.
|
|
133
|
+
parentId: dirname(dirent._path) == '.' ? sync.item : _items.get(dirname(dirent._path))?.id,
|
|
111
134
|
name: dirent.name,
|
|
112
135
|
};
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
process.stdout.write('(dry run) ');
|
|
117
|
-
else if (dirent.isDirectory()) {
|
|
118
|
-
const dir = await uploadItem(new Blob([], { type: 'inode/directory' }), uploadOpts);
|
|
119
|
-
_items.set(dirent._path, Object.assign(pick(dir, 'id', 'modifiedAt'), { path: dirent._path, hash: null }));
|
|
120
|
-
}
|
|
121
|
-
else {
|
|
122
|
-
const type = mime.getType(dirent._path) || 'application/octet-stream';
|
|
123
|
-
const content = fs.readFileSync(join(sync.localPath, dirent._path));
|
|
124
|
-
const file = await uploadItem(new Blob([content], { type }), uploadOpts);
|
|
125
|
-
_items.set(dirent._path, Object.assign(pick(file, 'id', 'modifiedAt', 'hash'), { path: dirent._path }));
|
|
126
|
-
}
|
|
127
|
-
io.done();
|
|
136
|
+
if (dirent.isDirectory()) {
|
|
137
|
+
const dir = await uploadItem(new Blob([], { type: 'inode/directory' }), uploadOpts);
|
|
138
|
+
_items.set(dirent._path, Object.assign(pick(dir, 'id', 'modifiedAt'), { path: dirent._path, hash: null }));
|
|
128
139
|
}
|
|
129
|
-
|
|
130
|
-
|
|
140
|
+
else {
|
|
141
|
+
const type = mime.getType(dirent._path) || 'application/octet-stream';
|
|
142
|
+
const content = fs.readFileSync(join(sync.local_path, dirent._path));
|
|
143
|
+
const file = await uploadItem(new Blob([content], { type }), uploadOpts);
|
|
144
|
+
_items.set(dirent._path, Object.assign(pick(file, 'id', 'modifiedAt', 'hash'), { path: dirent._path }));
|
|
131
145
|
}
|
|
146
|
+
});
|
|
147
|
+
if (!delta.remote_only.length) {
|
|
148
|
+
// Nothing
|
|
149
|
+
}
|
|
150
|
+
else if (opt.delete == 'remote') {
|
|
151
|
+
delta.remote_only.reverse();
|
|
152
|
+
if (!opt.verbose)
|
|
153
|
+
io.start('Deleting remote items');
|
|
132
154
|
}
|
|
133
|
-
if (opt.
|
|
134
|
-
|
|
135
|
-
|
|
155
|
+
else if (!opt.verbose)
|
|
156
|
+
io.start('Downloading remote items');
|
|
157
|
+
await applyAction(opt, delta.remote_only, item => (!opt.verbose ? '' : opt.delete == 'remote' ? 'Deleting remote ' : 'Downloading ') + item.path, async (item) => {
|
|
136
158
|
if (opt.delete == 'remote') {
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
process.stdout.write('(dry run) ');
|
|
141
|
-
else {
|
|
142
|
-
await deleteItem(item.id);
|
|
143
|
-
_items.delete(item.path);
|
|
144
|
-
}
|
|
145
|
-
io.done();
|
|
146
|
-
}
|
|
147
|
-
catch (e) {
|
|
148
|
-
io.error(e.message);
|
|
149
|
-
}
|
|
150
|
-
continue;
|
|
159
|
+
await deleteItem(item.id);
|
|
160
|
+
_items.delete(item.path);
|
|
161
|
+
return;
|
|
151
162
|
}
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
if (opt.dryRun)
|
|
156
|
-
process.stdout.write('(dry run) ');
|
|
157
|
-
else if (!item.hash) {
|
|
158
|
-
fs.mkdirSync(fullPath, { recursive: true });
|
|
159
|
-
}
|
|
160
|
-
else {
|
|
161
|
-
const blob = await downloadItem(item.id);
|
|
162
|
-
const content = await blob.bytes();
|
|
163
|
-
fs.writeFileSync(fullPath, content);
|
|
164
|
-
}
|
|
165
|
-
io.done();
|
|
163
|
+
const fullPath = join(sync.local_path, item.path);
|
|
164
|
+
if (!item.hash) {
|
|
165
|
+
fs.mkdirSync(fullPath, { recursive: true });
|
|
166
166
|
}
|
|
167
|
-
|
|
168
|
-
|
|
167
|
+
else {
|
|
168
|
+
const blob = await downloadItem(item.id);
|
|
169
|
+
const content = await blob.bytes();
|
|
170
|
+
fs.writeFileSync(fullPath, content);
|
|
169
171
|
}
|
|
170
|
-
}
|
|
171
|
-
|
|
172
|
-
io.start('
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
const
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
else {
|
|
180
|
-
const blob = await downloadItem(item.id);
|
|
181
|
-
const content = await blob.bytes();
|
|
182
|
-
fs.writeFileSync(join(sync.localPath, item.path), content);
|
|
183
|
-
}
|
|
184
|
-
console.log('server.');
|
|
185
|
-
}
|
|
186
|
-
else {
|
|
187
|
-
if (opt.dryRun)
|
|
188
|
-
process.stdout.write('(dry run) ');
|
|
189
|
-
else {
|
|
190
|
-
const updated = await updateItem(item.id, new Blob([content], { type }));
|
|
191
|
-
_items.set(item.path, Object.assign(pick(updated, 'id', 'modifiedAt', 'hash'), { path: item.path }));
|
|
192
|
-
}
|
|
193
|
-
console.log('local.');
|
|
194
|
-
}
|
|
172
|
+
});
|
|
173
|
+
if (!opt.verbose && delta.modified.length)
|
|
174
|
+
io.start('Syncing modified items');
|
|
175
|
+
await applyAction(opt, delta.modified, item => (opt.verbose ? 'Updating ' : '') + item.path, async (item) => {
|
|
176
|
+
if (item.modifiedAt.getTime() > fs.statSync(join(sync.local_path, item.path)).mtime.getTime()) {
|
|
177
|
+
const blob = await downloadItem(item.id);
|
|
178
|
+
const content = await blob.bytes();
|
|
179
|
+
fs.writeFileSync(join(sync.local_path, item.path), content);
|
|
180
|
+
return 'server.';
|
|
195
181
|
}
|
|
196
|
-
|
|
197
|
-
|
|
182
|
+
else {
|
|
183
|
+
const type = mime.getType(item.path) || 'application/octet-stream';
|
|
184
|
+
const content = fs.readFileSync(join(sync.local_path, item.path));
|
|
185
|
+
const updated = await updateItem(item.id, new Blob([content], { type }));
|
|
186
|
+
_items.set(item.path, Object.assign(pick(updated, 'id', 'modifiedAt', 'hash'), { path: item.path }));
|
|
187
|
+
return 'local.';
|
|
198
188
|
}
|
|
199
|
-
}
|
|
189
|
+
});
|
|
200
190
|
if (opt.dryRun)
|
|
201
|
-
return;
|
|
202
|
-
setItems(sync.
|
|
203
|
-
sync.
|
|
191
|
+
return stats;
|
|
192
|
+
setItems(sync.item, Array.from(_items.values()));
|
|
193
|
+
sync.last_synced = new Date();
|
|
204
194
|
saveConfig();
|
|
195
|
+
return stats;
|
|
205
196
|
}
|
package/dist/common.d.ts
CHANGED
|
@@ -64,13 +64,14 @@ export interface StorageLimits {
|
|
|
64
64
|
/** The maximum storage size per user in MB */
|
|
65
65
|
user_size: number;
|
|
66
66
|
}
|
|
67
|
-
export interface
|
|
68
|
-
|
|
69
|
-
|
|
67
|
+
export interface StorageStats {
|
|
68
|
+
usedBytes: number;
|
|
69
|
+
itemCount: number;
|
|
70
|
+
lastModified: Date;
|
|
71
|
+
lastTrashed: Date | null;
|
|
70
72
|
}
|
|
71
|
-
export interface UserStorageInfo {
|
|
73
|
+
export interface UserStorageInfo extends StorageStats {
|
|
72
74
|
limits: StorageLimits;
|
|
73
|
-
usage: StorageUsage;
|
|
74
75
|
}
|
|
75
76
|
export interface UserStorage extends UserStorageInfo {
|
|
76
77
|
items: StorageItemMetadata[];
|
|
@@ -82,24 +83,50 @@ export declare const StorageItemUpdate: z.ZodObject<{
|
|
|
82
83
|
name: z.ZodOptional<z.ZodString>;
|
|
83
84
|
owner: z.ZodOptional<z.ZodUUID>;
|
|
84
85
|
trash: z.ZodOptional<z.ZodBoolean>;
|
|
85
|
-
publicPermission: z.ZodOptional<z.
|
|
86
|
+
publicPermission: z.ZodOptional<z.ZodEnum<{
|
|
87
|
+
readonly None: 0;
|
|
88
|
+
readonly Read: 1;
|
|
89
|
+
readonly Comment: 2;
|
|
90
|
+
readonly Edit: 3;
|
|
91
|
+
readonly Manage: 5;
|
|
92
|
+
}> & {
|
|
93
|
+
readonly None: 0;
|
|
94
|
+
readonly Read: 1;
|
|
95
|
+
readonly Comment: 2;
|
|
96
|
+
readonly Edit: 3;
|
|
97
|
+
readonly Manage: 5;
|
|
98
|
+
}>;
|
|
86
99
|
}, z.core.$strip>;
|
|
87
100
|
export type StorageItemUpdate = z.infer<typeof StorageItemUpdate>;
|
|
88
|
-
export
|
|
89
|
-
createdAt:
|
|
90
|
-
dataURL:
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
101
|
+
export declare const StorageItemMetadata: z.ZodObject<{
|
|
102
|
+
createdAt: z.ZodCoercedDate<unknown>;
|
|
103
|
+
dataURL: z.ZodString;
|
|
104
|
+
hash: z.ZodOptional<z.ZodNullable<z.ZodString>>;
|
|
105
|
+
id: z.ZodUUID;
|
|
106
|
+
immutable: z.ZodBoolean;
|
|
107
|
+
modifiedAt: z.ZodCoercedDate<unknown>;
|
|
108
|
+
name: z.ZodString;
|
|
109
|
+
userId: z.ZodUUID;
|
|
110
|
+
parentId: z.ZodNullable<z.ZodUUID>;
|
|
111
|
+
publicPermission: z.ZodEnum<{
|
|
112
|
+
readonly None: 0;
|
|
113
|
+
readonly Read: 1;
|
|
114
|
+
readonly Comment: 2;
|
|
115
|
+
readonly Edit: 3;
|
|
116
|
+
readonly Manage: 5;
|
|
117
|
+
}> & {
|
|
118
|
+
readonly None: 0;
|
|
119
|
+
readonly Read: 1;
|
|
120
|
+
readonly Comment: 2;
|
|
121
|
+
readonly Edit: 3;
|
|
122
|
+
readonly Manage: 5;
|
|
123
|
+
};
|
|
124
|
+
size: z.ZodInt;
|
|
125
|
+
trashedAt: z.ZodNullable<z.ZodCoercedDate<unknown>>;
|
|
126
|
+
type: z.ZodString;
|
|
127
|
+
metadata: z.ZodRecord<z.ZodString, z.ZodUnknown>;
|
|
128
|
+
}, z.core.$strip>;
|
|
129
|
+
export interface StorageItemMetadata<T extends Record<string, unknown> = Record<string, unknown>> extends z.infer<typeof StorageItemMetadata> {
|
|
103
130
|
metadata: T;
|
|
104
131
|
}
|
|
105
132
|
/**
|
|
@@ -121,7 +148,19 @@ export declare const StorageBatchUpdate: z.ZodObject<{
|
|
|
121
148
|
name: z.ZodOptional<z.ZodString>;
|
|
122
149
|
owner: z.ZodOptional<z.ZodUUID>;
|
|
123
150
|
trash: z.ZodOptional<z.ZodBoolean>;
|
|
124
|
-
publicPermission: z.ZodOptional<z.
|
|
151
|
+
publicPermission: z.ZodOptional<z.ZodEnum<{
|
|
152
|
+
readonly None: 0;
|
|
153
|
+
readonly Read: 1;
|
|
154
|
+
readonly Comment: 2;
|
|
155
|
+
readonly Edit: 3;
|
|
156
|
+
readonly Manage: 5;
|
|
157
|
+
}> & {
|
|
158
|
+
readonly None: 0;
|
|
159
|
+
readonly Read: 1;
|
|
160
|
+
readonly Comment: 2;
|
|
161
|
+
readonly Edit: 3;
|
|
162
|
+
readonly Manage: 5;
|
|
163
|
+
}>;
|
|
125
164
|
}, z.core.$strip>>;
|
|
126
165
|
content: z.ZodRecord<z.ZodUUID, z.ZodObject<{
|
|
127
166
|
offset: z.ZodInt;
|
package/dist/common.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { Permission } from '@axium/core/access';
|
|
1
2
|
import * as z from 'zod';
|
|
2
3
|
export const syncProtocolVersion = 0;
|
|
3
4
|
/**
|
|
@@ -8,9 +9,26 @@ export const StorageItemUpdate = z
|
|
|
8
9
|
name: z.string(),
|
|
9
10
|
owner: z.uuid(),
|
|
10
11
|
trash: z.boolean(),
|
|
11
|
-
publicPermission:
|
|
12
|
+
publicPermission: Permission,
|
|
12
13
|
})
|
|
13
14
|
.partial();
|
|
15
|
+
export const StorageItemMetadata = z.object({
|
|
16
|
+
createdAt: z.coerce.date(),
|
|
17
|
+
dataURL: z.string(),
|
|
18
|
+
/** The hash of the file, or null if it is a directory */
|
|
19
|
+
hash: z.string().nullish(),
|
|
20
|
+
id: z.uuid(),
|
|
21
|
+
immutable: z.boolean(),
|
|
22
|
+
modifiedAt: z.coerce.date(),
|
|
23
|
+
name: z.string(),
|
|
24
|
+
userId: z.uuid(),
|
|
25
|
+
parentId: z.uuid().nullable(),
|
|
26
|
+
publicPermission: Permission,
|
|
27
|
+
size: z.int().nonnegative(),
|
|
28
|
+
trashedAt: z.coerce.date().nullable(),
|
|
29
|
+
type: z.string(),
|
|
30
|
+
metadata: z.record(z.string(), z.unknown()),
|
|
31
|
+
});
|
|
14
32
|
/**
|
|
15
33
|
* Formats:
|
|
16
34
|
*
|
package/dist/node.d.ts
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { StorageItemMetadata } from './common.js';
|
|
2
|
+
export declare function colorItem(item: StorageItemMetadata): string;
|
|
3
|
+
interface FormatItemsConfig {
|
|
4
|
+
items: (StorageItemMetadata & {
|
|
5
|
+
__size?: string;
|
|
6
|
+
})[];
|
|
7
|
+
users: Record<string, {
|
|
8
|
+
name: string;
|
|
9
|
+
}>;
|
|
10
|
+
humanReadable: boolean;
|
|
11
|
+
}
|
|
12
|
+
export declare function formatItems({ items, users, humanReadable }: FormatItemsConfig): Generator<string>;
|
|
13
|
+
export {};
|
package/dist/node.js
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import { formatBytes } from '@axium/core/format';
|
|
2
|
+
import { styleText } from 'node:util';
|
|
3
|
+
const executables = ['application/x-pie-executable', 'application/x-sharedlib', 'application/vnd.microsoft.portable-executable'];
|
|
4
|
+
const archives = [
|
|
5
|
+
'application/gzip',
|
|
6
|
+
'application/vnd.rar',
|
|
7
|
+
'application/x-7z-compressed',
|
|
8
|
+
'application/x-bzip',
|
|
9
|
+
'application/x-bzip2',
|
|
10
|
+
'application/x-tar',
|
|
11
|
+
'application/x-xz',
|
|
12
|
+
'application/zip',
|
|
13
|
+
];
|
|
14
|
+
export function colorItem(item) {
|
|
15
|
+
const { type, name } = item;
|
|
16
|
+
if (type === 'inode/directory')
|
|
17
|
+
return styleText('blue', name);
|
|
18
|
+
if (type.startsWith('image/') || type.startsWith('video/'))
|
|
19
|
+
return styleText('magenta', name);
|
|
20
|
+
if (type.startsWith('audio/'))
|
|
21
|
+
return styleText('cyan', name);
|
|
22
|
+
if (executables.includes(type))
|
|
23
|
+
return styleText('green', name);
|
|
24
|
+
if (archives.includes(type))
|
|
25
|
+
return styleText('red', name);
|
|
26
|
+
return name;
|
|
27
|
+
}
|
|
28
|
+
const publicPermString = ['---', 'r--', 'r-x', 'rwx', 'rwx', 'rwx'];
|
|
29
|
+
const __formatter = new Intl.DateTimeFormat('en-US', {
|
|
30
|
+
year: 'numeric',
|
|
31
|
+
month: 'short',
|
|
32
|
+
day: '2-digit',
|
|
33
|
+
hour12: false,
|
|
34
|
+
hour: '2-digit',
|
|
35
|
+
minute: '2-digit',
|
|
36
|
+
});
|
|
37
|
+
const formatDate = __formatter.format.bind(__formatter);
|
|
38
|
+
export function* formatItems({ items, users, humanReadable }) {
|
|
39
|
+
let sizeWidth = 0, nameWidth = 0;
|
|
40
|
+
for (const item of items) {
|
|
41
|
+
item.__size = item.type == 'inode/directory' ? '-' : humanReadable ? formatBytes(item.size) : item.size.toString();
|
|
42
|
+
sizeWidth = Math.max(sizeWidth, item.__size.length);
|
|
43
|
+
nameWidth = Math.max(nameWidth, users[item.userId].name.length);
|
|
44
|
+
}
|
|
45
|
+
for (const item of items) {
|
|
46
|
+
const owner = users[item.userId].name;
|
|
47
|
+
const type = item.type == 'inode/directory' ? 'd' : '-';
|
|
48
|
+
const ownerPerm = `r${item.immutable ? '-' : 'w'}x`;
|
|
49
|
+
const publicPerm = publicPermString[item.publicPermission];
|
|
50
|
+
yield `${type}${ownerPerm}${ownerPerm}${publicPerm}. ${owner.padEnd(nameWidth)} ${item.__size.padStart(sizeWidth)} ${formatDate(item.modifiedAt)} ${colorItem(item)}`;
|
|
51
|
+
}
|
|
52
|
+
}
|
package/dist/server/api.js
CHANGED
|
@@ -9,7 +9,7 @@ import * as z from 'zod';
|
|
|
9
9
|
import { batchFormatVersion, StorageItemUpdate, syncProtocolVersion } from '../common.js';
|
|
10
10
|
import '../polyfills.js';
|
|
11
11
|
import { getLimits } from './config.js';
|
|
12
|
-
import {
|
|
12
|
+
import { getUserStats, deleteRecursive, getRecursive, parseItem } from './db.js';
|
|
13
13
|
addRoute({
|
|
14
14
|
path: '/api/storage',
|
|
15
15
|
OPTIONS() {
|
|
@@ -106,20 +106,20 @@ addRoute({
|
|
|
106
106
|
error(503, 'User storage is disabled');
|
|
107
107
|
const userId = params.id;
|
|
108
108
|
await checkAuthForUser(request, userId);
|
|
109
|
-
const [
|
|
110
|
-
return {
|
|
109
|
+
const [stats, limits] = await Promise.all([getUserStats(userId), getLimits(userId)]).catch(withError('Could not fetch data'));
|
|
110
|
+
return Object.assign(stats, { limits });
|
|
111
111
|
},
|
|
112
112
|
async GET(request, params) {
|
|
113
113
|
if (!config.storage.enabled)
|
|
114
114
|
error(503, 'User storage is disabled');
|
|
115
115
|
const userId = params.id;
|
|
116
116
|
await checkAuthForUser(request, userId);
|
|
117
|
-
const [items,
|
|
117
|
+
const [items, stats, limits] = await Promise.all([
|
|
118
118
|
database.selectFrom('storage').where('userId', '=', userId).where('trashedAt', 'is', null).selectAll().execute(),
|
|
119
|
-
|
|
119
|
+
getUserStats(userId),
|
|
120
120
|
getLimits(userId),
|
|
121
121
|
]).catch(withError('Could not fetch data'));
|
|
122
|
-
return {
|
|
122
|
+
return Object.assign(stats, { limits, items: items.map(parseItem) });
|
|
123
123
|
},
|
|
124
124
|
});
|
|
125
125
|
addRoute({
|
package/dist/server/batch.js
CHANGED
|
@@ -13,7 +13,7 @@ import { join } from 'node:path/posix';
|
|
|
13
13
|
import * as z from 'zod';
|
|
14
14
|
import { StorageBatchUpdate } from '../common.js';
|
|
15
15
|
import { getLimits } from './config.js';
|
|
16
|
-
import {
|
|
16
|
+
import { getUserStats, getRecursiveIds, parseItem } from './db.js';
|
|
17
17
|
addRoute({
|
|
18
18
|
path: '/api/storage/batch',
|
|
19
19
|
async POST(req) {
|
|
@@ -27,7 +27,7 @@ addRoute({
|
|
|
27
27
|
const { userId, user } = await getSessionAndUser(token).catch(withError('Invalid session token', 401));
|
|
28
28
|
if (user.isSuspended)
|
|
29
29
|
error(403, 'User is suspended');
|
|
30
|
-
const [usage, limits] = await Promise.all([
|
|
30
|
+
const [usage, limits] = await Promise.all([getUserStats(userId), getLimits(userId)]).catch(withError('Could not fetch usage and/or limits'));
|
|
31
31
|
const batchHeaderSize = Number(req.headers.get('x-batch-header-size'));
|
|
32
32
|
if (!Number.isSafeInteger(batchHeaderSize) || batchHeaderSize < 2)
|
|
33
33
|
error(400, 'Invalid or missing header, X-Batch-Header-Size');
|
|
@@ -92,7 +92,7 @@ addRoute({
|
|
|
92
92
|
continue;
|
|
93
93
|
error(403, 'Missing permission for item: ' + item.id);
|
|
94
94
|
}
|
|
95
|
-
if (limits.user_size && (usage.
|
|
95
|
+
if (limits.user_size && (usage.usedBytes + size - items.reduce((sum, item) => sum + item.size, 0)) / 1_000_000 >= limits.user_size)
|
|
96
96
|
error(413, 'Not enough space');
|
|
97
97
|
const tx = await database.startTransaction().execute();
|
|
98
98
|
const results = new Map();
|
package/dist/server/db.d.ts
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { type Schema } from '@axium/server/database';
|
|
2
2
|
import type { Generated, Selectable } from 'kysely';
|
|
3
|
-
import type { StorageItemMetadata,
|
|
3
|
+
import type { StorageItemMetadata, StorageStats } from '../common.js';
|
|
4
4
|
import '../polyfills.js';
|
|
5
|
+
import type { Permission } from '@axium/core';
|
|
5
6
|
declare module '@axium/server/database' {
|
|
6
7
|
interface Schema {
|
|
7
8
|
storage: {
|
|
@@ -16,7 +17,7 @@ declare module '@axium/server/database' {
|
|
|
16
17
|
trashedAt: Date | null;
|
|
17
18
|
type: string;
|
|
18
19
|
userId: string;
|
|
19
|
-
publicPermission: Generated<
|
|
20
|
+
publicPermission: Generated<Permission>;
|
|
20
21
|
metadata: Generated<Record<string, unknown>>;
|
|
21
22
|
};
|
|
22
23
|
}
|
|
@@ -36,7 +37,7 @@ export declare function parseItem<T extends SelectedItem>(item: T): Omit<T, keyo
|
|
|
36
37
|
/**
|
|
37
38
|
* Returns the current usage of the storage for a user in bytes.
|
|
38
39
|
*/
|
|
39
|
-
export declare function
|
|
40
|
+
export declare function getUserStats(userId: string): Promise<StorageStats>;
|
|
40
41
|
export declare function get(itemId: string): Promise<StorageItemMetadata>;
|
|
41
42
|
export declare function getRecursive(this: {
|
|
42
43
|
path: string;
|
package/dist/server/db.js
CHANGED
|
@@ -29,15 +29,19 @@ export function parseItem(item) {
|
|
|
29
29
|
/**
|
|
30
30
|
* Returns the current usage of the storage for a user in bytes.
|
|
31
31
|
*/
|
|
32
|
-
export async function
|
|
32
|
+
export async function getUserStats(userId) {
|
|
33
33
|
const result = await database
|
|
34
34
|
.selectFrom('storage')
|
|
35
35
|
.where('userId', '=', userId)
|
|
36
|
-
.select(eb =>
|
|
37
|
-
|
|
36
|
+
.select(eb => [
|
|
37
|
+
eb.fn.countAll().as('itemCount'),
|
|
38
|
+
eb.fn.sum('size').as('usedBytes'),
|
|
39
|
+
eb.fn.max('modifiedAt').as('lastModified'),
|
|
40
|
+
eb.fn.max('trashedAt').as('lastTrashed'),
|
|
41
|
+
])
|
|
38
42
|
.executeTakeFirstOrThrow();
|
|
39
|
-
result.
|
|
40
|
-
result.
|
|
43
|
+
result.usedBytes = Number(result.usedBytes || 0);
|
|
44
|
+
result.itemCount = Number(result.itemCount);
|
|
41
45
|
return result;
|
|
42
46
|
}
|
|
43
47
|
export async function get(itemId) {
|
package/dist/server/raw.js
CHANGED
|
@@ -11,7 +11,7 @@ import { join } from 'node:path/posix';
|
|
|
11
11
|
import * as z from 'zod';
|
|
12
12
|
import '../polyfills.js';
|
|
13
13
|
import { defaultCASMime, getLimits } from './config.js';
|
|
14
|
-
import {
|
|
14
|
+
import { getUserStats, parseItem } from './db.js';
|
|
15
15
|
addRoute({
|
|
16
16
|
path: '/raw/storage',
|
|
17
17
|
async PUT(request) {
|
|
@@ -21,7 +21,7 @@ addRoute({
|
|
|
21
21
|
if (!token)
|
|
22
22
|
error(401, 'Missing session token');
|
|
23
23
|
const { userId } = await getSessionAndUser(token).catch(withError('Invalid session token', 401));
|
|
24
|
-
const [usage, limits] = await Promise.all([
|
|
24
|
+
const [usage, limits] = await Promise.all([getUserStats(userId), getLimits(userId)]).catch(withError('Could not fetch usage and/or limits'));
|
|
25
25
|
const name = request.headers.get('x-name');
|
|
26
26
|
if (!name)
|
|
27
27
|
error(400, 'Missing name header');
|
|
@@ -39,9 +39,9 @@ addRoute({
|
|
|
39
39
|
const size = Number(request.headers.get('content-length'));
|
|
40
40
|
if (Number.isNaN(size))
|
|
41
41
|
error(411, 'Missing or invalid content length header');
|
|
42
|
-
if (limits.user_items && usage.
|
|
42
|
+
if (limits.user_items && usage.itemCount >= limits.user_items)
|
|
43
43
|
error(409, 'Too many items');
|
|
44
|
-
if (limits.user_size && (usage.
|
|
44
|
+
if (limits.user_size && (usage.usedBytes + size) / 1_000_000 >= limits.user_size)
|
|
45
45
|
error(413, 'Not enough space');
|
|
46
46
|
if (limits.item_size && size > limits.item_size * 1_000_000)
|
|
47
47
|
error(413, 'File size exceeds maximum size');
|
|
@@ -132,8 +132,8 @@ addRoute({
|
|
|
132
132
|
const size = Number(request.headers.get('content-length'));
|
|
133
133
|
if (Number.isNaN(size))
|
|
134
134
|
error(411, 'Missing or invalid content length header');
|
|
135
|
-
const [usage, limits] = await Promise.all([
|
|
136
|
-
if (limits.user_size && (usage.
|
|
135
|
+
const [usage, limits] = await Promise.all([getUserStats(item.userId), getLimits(item.userId)]).catch(withError('Could not fetch usage and/or limits'));
|
|
136
|
+
if (limits.user_size && (usage.usedBytes + size - item.size) / 1_000_000 >= limits.user_size)
|
|
137
137
|
error(413, 'Not enough space');
|
|
138
138
|
if (limits.item_size && size > limits.item_size * 1_000_000)
|
|
139
139
|
error(413, 'File size exceeds maximum size');
|
package/lib/Usage.svelte
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
<script lang="ts">
|
|
2
2
|
import { formatBytes } from '@axium/core/format';
|
|
3
3
|
import { NumberBar } from '@axium/client/components';
|
|
4
|
-
import {
|
|
4
|
+
import { getUserStats } from '@axium/storage/client';
|
|
5
5
|
import type { UserStorageInfo } from '@axium/storage/common';
|
|
6
6
|
|
|
7
7
|
const { userId, info }: { userId?: string; info?: UserStorageInfo } = $props();
|
|
@@ -10,13 +10,13 @@
|
|
|
10
10
|
{#if !info && !userId}
|
|
11
11
|
<p>Log in to see storage usage.</p>
|
|
12
12
|
{:else}
|
|
13
|
-
{#await info ||
|
|
13
|
+
{#await info || getUserStats(userId!) then info}
|
|
14
14
|
<p>
|
|
15
15
|
<a href="/files/usage">
|
|
16
16
|
<NumberBar
|
|
17
17
|
max={info.limits.user_size && info.limits.user_size * 1_000_000}
|
|
18
|
-
value={info.
|
|
19
|
-
text="Using {formatBytes(info.
|
|
18
|
+
value={info.usedBytes}
|
|
19
|
+
text="Using {formatBytes(info.usedBytes)} {!info.limits.user_size
|
|
20
20
|
? ''
|
|
21
21
|
: 'of ' + formatBytes(info.limits.user_size * 1_000_000)}"
|
|
22
22
|
/>
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@axium/storage",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.9.0",
|
|
4
4
|
"author": "James Prevett <axium@jamespre.dev>",
|
|
5
5
|
"description": "User file storage for Axium",
|
|
6
6
|
"funding": {
|
|
@@ -39,7 +39,7 @@
|
|
|
39
39
|
},
|
|
40
40
|
"peerDependencies": {
|
|
41
41
|
"@axium/client": ">=0.6.0",
|
|
42
|
-
"@axium/core": ">=0.
|
|
42
|
+
"@axium/core": ">=0.10.0",
|
|
43
43
|
"@axium/server": ">=0.26.0",
|
|
44
44
|
"@sveltejs/kit": "^2.27.3",
|
|
45
45
|
"utilium": "^2.3.8"
|
|
@@ -8,12 +8,10 @@
|
|
|
8
8
|
const { limits } = data.info;
|
|
9
9
|
|
|
10
10
|
let items = $state(data.info.items.filter(i => i.type != 'inode/directory').sort((a, b) => Math.sign(b.size - a.size)));
|
|
11
|
-
const
|
|
11
|
+
const usedBytes = $state(data.info.usedBytes);
|
|
12
12
|
|
|
13
13
|
let dialogs = $state<Record<string, HTMLDialogElement>>({});
|
|
14
|
-
let barText = $derived(
|
|
15
|
-
`Using ${formatBytes(usage?.bytes)} ${limits.user_size ? 'of ' + formatBytes(limits.user_size * 1_000_000) : ''}`
|
|
16
|
-
);
|
|
14
|
+
let barText = $derived(`Using ${formatBytes(usedBytes)} ${limits.user_size ? 'of ' + formatBytes(limits.user_size * 1_000_000) : ''}`);
|
|
17
15
|
</script>
|
|
18
16
|
|
|
19
17
|
<svelte:head>
|
|
@@ -28,6 +26,6 @@
|
|
|
28
26
|
|
|
29
27
|
<h2>Storage Usage</h2>
|
|
30
28
|
|
|
31
|
-
<p><NumberBar max={limits.user_size * 1_000_000} value={
|
|
29
|
+
<p><NumberBar max={limits.user_size * 1_000_000} value={usedBytes} text={barText} /></p>
|
|
32
30
|
|
|
33
31
|
<StorageList bind:items emptyText="You have not uploaded any files yet." />
|
package/dist/client/paths.d.ts
DELETED
package/dist/client/paths.js
DELETED
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
export function walkItems(path, items) {
|
|
2
|
-
const resolved = [];
|
|
3
|
-
let currentItem = null, currentParentId = null;
|
|
4
|
-
const target = path.split('/').filter(p => p);
|
|
5
|
-
for (const part of target) {
|
|
6
|
-
for (const item of items) {
|
|
7
|
-
if (item.parentId != currentParentId || item.name != part)
|
|
8
|
-
continue;
|
|
9
|
-
currentItem = item;
|
|
10
|
-
currentParentId = item.id;
|
|
11
|
-
resolved.push(part);
|
|
12
|
-
break;
|
|
13
|
-
}
|
|
14
|
-
}
|
|
15
|
-
return currentItem;
|
|
16
|
-
}
|