@augmentd-labs/canvas-edge 0.0.0-stage → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +661 -0
- package/NOTICE +112 -0
- package/README.md +76 -2
- package/bin/canvas-edge +2 -0
- package/bin/healthcheck.mjs +9 -0
- package/node_modules/canvas-stored/LICENSE +661 -0
- package/node_modules/canvas-stored/NOTICE +85 -0
- package/node_modules/canvas-stored/README.md +472 -0
- package/node_modules/canvas-stored/package.json +26 -0
- package/node_modules/canvas-stored/src/backends/BackendManager.js +60 -0
- package/node_modules/canvas-stored/src/backends/StorageBackend.js +72 -0
- package/node_modules/canvas-stored/src/backends/cacache/index.js +130 -0
- package/node_modules/canvas-stored/src/backends/canvas/index.js +722 -0
- package/node_modules/canvas-stored/src/backends/file/index.js +658 -0
- package/node_modules/canvas-stored/src/backends/gdrive/index.js +924 -0
- package/node_modules/canvas-stored/src/backends/http/index.js +47 -0
- package/node_modules/canvas-stored/src/backends/s3/index.js +25 -0
- package/node_modules/canvas-stored/src/cache/index.js +65 -0
- package/node_modules/canvas-stored/src/extractors/audio.js +53 -0
- package/node_modules/canvas-stored/src/extractors/image.js +90 -0
- package/node_modules/canvas-stored/src/extractors/index.js +41 -0
- package/node_modules/canvas-stored/src/extractors/text.js +65 -0
- package/node_modules/canvas-stored/src/index/index.js +360 -0
- package/node_modules/canvas-stored/src/index.js +2058 -0
- package/node_modules/canvas-stored/src/sync/JobQueue.js +0 -0
- package/node_modules/canvas-stored/src/sync/Ledger.js +138 -0
- package/node_modules/canvas-stored/src/sync/Mirror.js +995 -0
- package/node_modules/canvas-stored/src/sync/SyncQueue.js +111 -0
- package/node_modules/canvas-stored/src/sync/keys.js +140 -0
- package/node_modules/canvas-stored/src/sync/worker.js +52 -0
- package/node_modules/canvas-stored/src/utils/checksum.js +32 -0
- package/node_modules/canvas-stored/src/utils/common.js +37 -0
- package/node_modules/canvas-stored/src/utils/mime.js +107 -0
- package/node_modules/canvas-stored/src/utils/mount.js +99 -0
- package/package.json +49 -4
- package/src/EdgeClient.js +129 -0
- package/src/control.js +48 -0
- package/src/daemon.js +88 -0
- package/src/env.js +182 -0
- package/src/fuse-runtime.js +173 -0
- package/src/index.js +26 -0
- package/src/mirror-runtime.js +232 -0
- package/src/remote-config.js +69 -0
- package/src/runtime.js +61 -0
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
import { Worker } from 'worker_threads';
|
|
2
|
+
import { EventEmitter } from 'events';
|
|
3
|
+
import cacache from 'cacache';
|
|
4
|
+
import Debug from 'debug';
|
|
5
|
+
|
|
6
|
+
const debug = Debug('stored:sync');
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Background sync queue for remote backend writes.
|
|
10
|
+
*
|
|
11
|
+
* Every job is `{ id, cacheRoot, cacheKey, targets[] }` — the bytes already
|
|
12
|
+
* sit in the cache under `cacheKey`; each target gets a copy. Two lanes:
|
|
13
|
+
* - `driver: 'file'` targets are streamed by a lazily-spawned worker thread
|
|
14
|
+
* (disk-to-disk copy is the one thing worth taking off the event loop);
|
|
15
|
+
* - every other driver is committed IN-PROCESS through its live backend
|
|
16
|
+
* (`backend.commit(key, cachePath)`): network uploads are async I/O, and
|
|
17
|
+
* the driver's credentials, token and path cache live on the main thread.
|
|
18
|
+
* One `synced` event per job carries the merged results of both lanes.
|
|
19
|
+
*/
|
|
20
|
+
export default class SyncQueue extends EventEmitter {
|
|
21
|
+
#worker = null;
|
|
22
|
+
#seq = 0;
|
|
23
|
+
#pending = new Map(); // seq → resolve(results) for in-flight worker jobs
|
|
24
|
+
#resolveBackend;
|
|
25
|
+
|
|
26
|
+
/** @param {{ resolveBackend?: (name:string) => object|undefined }} [options] */
|
|
27
|
+
constructor({ resolveBackend = null } = {}) {
|
|
28
|
+
super();
|
|
29
|
+
this.#resolveBackend = typeof resolveBackend === 'function' ? resolveBackend : () => undefined;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
enqueue(job) {
|
|
33
|
+
const targets = Array.isArray(job?.targets) ? job.targets : [];
|
|
34
|
+
const viaWorker = targets.filter((t) => t.driver === 'file');
|
|
35
|
+
const inProcess = targets.filter((t) => t.driver !== 'file');
|
|
36
|
+
const lanes = [];
|
|
37
|
+
if (viaWorker.length) lanes.push(this.#viaWorker({ ...job, targets: viaWorker }));
|
|
38
|
+
if (inProcess.length) lanes.push(this.#inProcess(job, inProcess));
|
|
39
|
+
if (!lanes.length) return;
|
|
40
|
+
|
|
41
|
+
Promise.all(lanes)
|
|
42
|
+
.then((parts) => {
|
|
43
|
+
const results = parts.flat();
|
|
44
|
+
debug(`Sync complete: ${String(job.id).slice(0, 19)}... → ${results.map((r) => `${r.backend}:${r.success}`).join(', ')}`);
|
|
45
|
+
this.emit('synced', { id: job.id, results });
|
|
46
|
+
})
|
|
47
|
+
.catch((err) => this.emit('error', err));
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
async stop() {
|
|
51
|
+
if (this.#worker) {
|
|
52
|
+
await this.#worker.terminate();
|
|
53
|
+
this.#worker = null;
|
|
54
|
+
}
|
|
55
|
+
// Jobs the worker never answered must not hang their `synced` forever.
|
|
56
|
+
for (const [seq, resolve] of this.#pending) {
|
|
57
|
+
this.#pending.delete(seq);
|
|
58
|
+
resolve([]);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
#viaWorker(job) {
|
|
63
|
+
if (!this.#worker) this.#spawnWorker();
|
|
64
|
+
const seq = ++this.#seq;
|
|
65
|
+
return new Promise((resolve) => {
|
|
66
|
+
this.#pending.set(seq, resolve);
|
|
67
|
+
this.#worker.postMessage({ seq, ...job });
|
|
68
|
+
});
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
async #inProcess({ cacheRoot, cacheKey }, targets) {
|
|
72
|
+
const info = await cacache.get.info(cacheRoot, cacheKey).catch(() => null);
|
|
73
|
+
if (!info) {
|
|
74
|
+
return targets.map((t) => ({ backend: t.name, key: t.key, success: false, error: 'Cache read failed: entry not found' }));
|
|
75
|
+
}
|
|
76
|
+
const results = [];
|
|
77
|
+
for (const target of targets) {
|
|
78
|
+
const backend = this.#resolveBackend(target.name);
|
|
79
|
+
if (!backend || typeof backend.commit !== 'function') {
|
|
80
|
+
results.push({ backend: target.name, key: target.key, success: false, error: `Backend not registered: ${target.name}` });
|
|
81
|
+
continue;
|
|
82
|
+
}
|
|
83
|
+
try {
|
|
84
|
+
await backend.commit(target.key, info.path);
|
|
85
|
+
results.push({ backend: target.name, key: target.key, success: true });
|
|
86
|
+
} catch (err) {
|
|
87
|
+
debug(`In-process sync to ${target.name}:${target.key} failed: ${err.message}`);
|
|
88
|
+
results.push({ backend: target.name, key: target.key, success: false, error: err.message });
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
return results;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
#spawnWorker() {
|
|
95
|
+
this.#worker = new Worker(new URL('./worker.js', import.meta.url));
|
|
96
|
+
this.#worker.on('message', (msg) => {
|
|
97
|
+
const resolve = this.#pending.get(msg.seq);
|
|
98
|
+
if (resolve) {
|
|
99
|
+
this.#pending.delete(msg.seq);
|
|
100
|
+
resolve(msg.results || []);
|
|
101
|
+
return;
|
|
102
|
+
}
|
|
103
|
+
// No correlation (foreign/legacy message) — surface as-is.
|
|
104
|
+
this.emit('synced', msg);
|
|
105
|
+
});
|
|
106
|
+
this.#worker.on('error', (err) => {
|
|
107
|
+
debug(`Worker error: ${err.message}`);
|
|
108
|
+
this.emit('error', err);
|
|
109
|
+
});
|
|
110
|
+
}
|
|
111
|
+
}
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
import picomatch from 'picomatch';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Key hygiene shared by the mirror engine and the canvas driver. Keys are
|
|
5
|
+
* relative, '/'-separated paths in NFC — the spelling the hub uses on the
|
|
6
|
+
* wire (`docs/sync-protocol.md`). Everything a device filesystem can produce
|
|
7
|
+
* that the hub (or the other platform) would refuse is caught here and
|
|
8
|
+
* surfaced as a `skip {reason}` instead of a silent loss.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
// Mirror-side defaults on top of the hub's effective exclusions.
|
|
12
|
+
export const MIRROR_IGNORE_DEFAULTS = Object.freeze([
|
|
13
|
+
'.workspace/**',
|
|
14
|
+
'.stored-tmp/**',
|
|
15
|
+
'**/.DS_Store',
|
|
16
|
+
'**/Thumbs.db',
|
|
17
|
+
'**/desktop.ini',
|
|
18
|
+
'**/~$*',
|
|
19
|
+
'**/*.tmp',
|
|
20
|
+
'**/*.part',
|
|
21
|
+
'**/*.crdownload',
|
|
22
|
+
'**/.~lock.*#',
|
|
23
|
+
'**/*.swp',
|
|
24
|
+
]);
|
|
25
|
+
|
|
26
|
+
const MAX_KEY_BYTES = 4096;
|
|
27
|
+
const MAX_SEGMENT_BYTES = 255;
|
|
28
|
+
const WINDOWS_RESERVED = /^(con|prn|aux|nul|com[1-9]|lpt[1-9])(\..*)?$/i;
|
|
29
|
+
// eslint-disable-next-line no-control-regex
|
|
30
|
+
const WINDOWS_ILLEGAL = /[<>:"|?*\x00-\x1f]/;
|
|
31
|
+
|
|
32
|
+
/** NFC + '/' separators, no empty / leading / trailing segments. */
|
|
33
|
+
export function normalizeKey(key) {
|
|
34
|
+
return String(key ?? '')
|
|
35
|
+
.normalize('NFC')
|
|
36
|
+
.replace(/\\/g, '/')
|
|
37
|
+
.split('/')
|
|
38
|
+
.filter((s) => s.length > 0)
|
|
39
|
+
.join('/');
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Why a (normalized) key cannot be synced, or null when it can. Reasons:
|
|
44
|
+
* `empty`, `nul`, `traversal`, `absolute`, `too-long`, `segment-too-long`,
|
|
45
|
+
* `reserved-name`, `illegal-chars`, `trailing-dot-or-space`.
|
|
46
|
+
*/
|
|
47
|
+
export function validateKey(key) {
|
|
48
|
+
if (typeof key !== 'string' || key.length === 0) return 'empty';
|
|
49
|
+
if (key.includes('\0')) return 'nul';
|
|
50
|
+
if (key.startsWith('/') || /^[a-zA-Z]:[\\/]/.test(key)) return 'absolute';
|
|
51
|
+
if (Buffer.byteLength(key, 'utf8') > MAX_KEY_BYTES) return 'too-long';
|
|
52
|
+
for (const seg of key.split('/')) {
|
|
53
|
+
if (seg === '' || seg === '.' || seg === '..') return 'traversal';
|
|
54
|
+
if (Buffer.byteLength(seg, 'utf8') > MAX_SEGMENT_BYTES) return 'segment-too-long';
|
|
55
|
+
if (WINDOWS_RESERVED.test(seg)) return 'reserved-name';
|
|
56
|
+
if (WINDOWS_ILLEGAL.test(seg)) return 'illegal-chars';
|
|
57
|
+
if (/[. ]$/.test(seg)) return 'trailing-dot-or-space';
|
|
58
|
+
}
|
|
59
|
+
return null;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
const matcherCache = new WeakMap();
|
|
63
|
+
function matcherFor(patterns) {
|
|
64
|
+
const list = (Array.isArray(patterns) ? patterns : [patterns]).filter((p) => typeof p === 'string' && p.trim());
|
|
65
|
+
if (!list.length) return null;
|
|
66
|
+
if (Array.isArray(patterns)) {
|
|
67
|
+
const cached = matcherCache.get(patterns);
|
|
68
|
+
if (cached) return cached;
|
|
69
|
+
}
|
|
70
|
+
// "dir/**" also prunes "dir" itself (same convention as the file driver).
|
|
71
|
+
const globs = list.flatMap((p) => (p.endsWith('/**') ? [p, p.slice(0, -3)] : [p]));
|
|
72
|
+
const match = picomatch(globs, { dot: true });
|
|
73
|
+
if (Array.isArray(patterns)) matcherCache.set(patterns, match);
|
|
74
|
+
return match;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** Does `key` match any of the ignore globs (picomatch, dotfiles included)? */
|
|
78
|
+
export function isIgnored(key, patterns = MIRROR_IGNORE_DEFAULTS) {
|
|
79
|
+
const k = normalizeKey(key);
|
|
80
|
+
if (!k) return false;
|
|
81
|
+
const match = matcherFor(patterns);
|
|
82
|
+
if (!match) return false;
|
|
83
|
+
if (match(k)) return true;
|
|
84
|
+
// A pattern naming a directory covers everything under it.
|
|
85
|
+
const parts = k.split('/');
|
|
86
|
+
for (let i = 1; i < parts.length; i += 1) {
|
|
87
|
+
if (match(parts.slice(0, i).join('/'))) return true;
|
|
88
|
+
}
|
|
89
|
+
return false;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* Selective sync: `prefixes` is a list of key prefixes (`Docs`, `Photos/2024/`)
|
|
94
|
+
* or picomatch globs. Empty/absent = everything. A prefix covers itself and
|
|
95
|
+
* the subtree under it.
|
|
96
|
+
*/
|
|
97
|
+
export function matchesPrefixes(key, prefixes) {
|
|
98
|
+
const list = (Array.isArray(prefixes) ? prefixes : [prefixes]).filter((p) => typeof p === 'string' && p.trim());
|
|
99
|
+
if (!list.length) return true;
|
|
100
|
+
const k = normalizeKey(key);
|
|
101
|
+
for (const raw of list) {
|
|
102
|
+
const p = normalizeKey(raw);
|
|
103
|
+
if (!p) return true;
|
|
104
|
+
if (/[*?[\]{}!]/.test(p)) {
|
|
105
|
+
// A glob covers the keys it matches and everything under a
|
|
106
|
+
// directory it matches (`Photos/20*` → `Photos/2024/a.jpg`).
|
|
107
|
+
const match = picomatch(p, { dot: true });
|
|
108
|
+
if (match(k)) return true;
|
|
109
|
+
const parts = k.split('/');
|
|
110
|
+
for (let i = 1; i < parts.length; i += 1) if (match(parts.slice(0, i).join('/'))) return true;
|
|
111
|
+
continue;
|
|
112
|
+
}
|
|
113
|
+
if (k === p || k.startsWith(`${p}/`)) return true;
|
|
114
|
+
}
|
|
115
|
+
return false;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
const pad = (n) => String(n).padStart(2, '0');
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* Dropbox-style conflict copy name:
|
|
122
|
+
* `<stem> (conflict from <device> <YYYY-MM-DD HHmm>).<ext>` (UTC), in the
|
|
123
|
+
* same directory as `key`. Matches canvas-fuse's `conflict_copy_key`.
|
|
124
|
+
*/
|
|
125
|
+
export function conflictKey(key, deviceName, date = new Date()) {
|
|
126
|
+
const k = normalizeKey(key);
|
|
127
|
+
const slash = k.lastIndexOf('/');
|
|
128
|
+
const dir = slash >= 0 ? k.slice(0, slash) : null;
|
|
129
|
+
const name = slash >= 0 ? k.slice(slash + 1) : k;
|
|
130
|
+
const dot = name.lastIndexOf('.');
|
|
131
|
+
const stem = dot > 0 ? name.slice(0, dot) : name;
|
|
132
|
+
const ext = dot > 0 ? name.slice(dot + 1) : null;
|
|
133
|
+
const at = date instanceof Date ? date : new Date(date);
|
|
134
|
+
const stamp = `${at.getUTCFullYear()}-${pad(at.getUTCMonth() + 1)}-${pad(at.getUTCDate())} ${pad(at.getUTCHours())}${pad(at.getUTCMinutes())}`;
|
|
135
|
+
const device = String(deviceName || 'device').replace(/[/\\]/g, '_');
|
|
136
|
+
const leaf = ext ? `${stem} (conflict from ${device} ${stamp}).${ext}` : `${stem} (conflict from ${device} ${stamp})`;
|
|
137
|
+
return dir ? `${dir}/${leaf}` : leaf;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
export default { normalizeKey, validateKey, isIgnored, matchesPrefixes, conflictKey, MIRROR_IGNORE_DEFAULTS };
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import { parentPort } from 'worker_threads';
|
|
2
|
+
import cacache from 'cacache';
|
|
3
|
+
import { createWriteStream } from 'fs';
|
|
4
|
+
import { mkdir } from 'fs/promises';
|
|
5
|
+
import { pipeline } from 'stream/promises';
|
|
6
|
+
import { dirname, join } from 'path';
|
|
7
|
+
|
|
8
|
+
// memoize:false — never hold blobs (up to 20GB) in the worker's RAM; stream
|
|
9
|
+
// straight from the cache content store to each target.
|
|
10
|
+
const CACHE_OPTS = { memoize: false };
|
|
11
|
+
|
|
12
|
+
parentPort.on('message', async ({ seq, id, cacheRoot, cacheKey, targets }) => {
|
|
13
|
+
const results = [];
|
|
14
|
+
|
|
15
|
+
// Confirm the entry exists before fanning out (cheap index lookup, no read).
|
|
16
|
+
const info = await cacache.get.info(cacheRoot, cacheKey).catch(() => null);
|
|
17
|
+
if (!info) {
|
|
18
|
+
for (const target of targets) {
|
|
19
|
+
results.push({ backend: target.name, success: false, error: 'Cache read failed: entry not found' });
|
|
20
|
+
}
|
|
21
|
+
parentPort.postMessage({ seq, id, results });
|
|
22
|
+
return;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
for (const target of targets) {
|
|
26
|
+
try {
|
|
27
|
+
switch (target.driver) {
|
|
28
|
+
case 'file': {
|
|
29
|
+
const filePath = join(target.root, target.key);
|
|
30
|
+
await mkdir(dirname(filePath), { recursive: true });
|
|
31
|
+
// Fresh read stream per target — streams are single-use and
|
|
32
|
+
// we never materialize the blob in memory.
|
|
33
|
+
await pipeline(
|
|
34
|
+
cacache.get.stream(cacheRoot, cacheKey, CACHE_OPTS),
|
|
35
|
+
createWriteStream(filePath),
|
|
36
|
+
);
|
|
37
|
+
results.push({ backend: target.name, key: target.key, success: true });
|
|
38
|
+
break;
|
|
39
|
+
}
|
|
40
|
+
// Non-file drivers (gdrive, …) are committed in-process by the
|
|
41
|
+
// SyncQueue through their live backend instance — they never
|
|
42
|
+
// reach this worker. Anything else here is a routing bug.
|
|
43
|
+
default:
|
|
44
|
+
results.push({ backend: target.name, success: false, error: `Unknown driver: ${target.driver}` });
|
|
45
|
+
}
|
|
46
|
+
} catch (err) {
|
|
47
|
+
results.push({ backend: target.name, success: false, error: err.message });
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
parentPort.postMessage({ seq, id, results });
|
|
52
|
+
});
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import crypto from 'crypto';
|
|
2
|
+
import fs from 'fs';
|
|
3
|
+
|
|
4
|
+
export function checksumBuffer(buffer, algorithms = ['sha256']) {
|
|
5
|
+
const checksums = {};
|
|
6
|
+
for (const algo of algorithms) {
|
|
7
|
+
checksums[algo] = crypto.createHash(algo).update(buffer).digest('hex');
|
|
8
|
+
}
|
|
9
|
+
return checksums;
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
export async function checksumStream(stream, algorithms = ['sha256']) {
|
|
13
|
+
const hashes = algorithms.map(algo => ({ algo, hash: crypto.createHash(algo) }));
|
|
14
|
+
|
|
15
|
+
for await (const chunk of stream) {
|
|
16
|
+
for (const { hash } of hashes) hash.update(chunk);
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
const checksums = {};
|
|
20
|
+
for (const { algo, hash } of hashes) {
|
|
21
|
+
checksums[algo] = hash.digest('hex');
|
|
22
|
+
}
|
|
23
|
+
return checksums;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export function checksumFile(filePath, algorithms = ['sha256']) {
|
|
27
|
+
return checksumStream(fs.createReadStream(filePath), algorithms);
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export function formatId(checksums, primaryAlgo = 'sha256') {
|
|
31
|
+
return `${primaryAlgo}:${checksums[primaryAlgo]}`;
|
|
32
|
+
}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import fs from 'fs';
|
|
2
|
+
import path from 'path';
|
|
3
|
+
|
|
4
|
+
const DEFAULT_ROOT = './.stored';
|
|
5
|
+
|
|
6
|
+
/** Resolve layout: `<root>/{index,cache,data}` with per-subdir overrides. */
|
|
7
|
+
export function resolveStoredPaths(config = {}) {
|
|
8
|
+
const root = path.resolve(config.root ?? DEFAULT_ROOT);
|
|
9
|
+
return {
|
|
10
|
+
root,
|
|
11
|
+
index: path.resolve(config.index?.path ?? path.join(root, 'index')),
|
|
12
|
+
cache: path.resolve(config.cache?.path ?? path.join(root, 'cache')),
|
|
13
|
+
data: path.resolve(config.data?.path ?? path.join(root, 'data')),
|
|
14
|
+
};
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export function isJson(input) {
|
|
18
|
+
if (typeof input !== 'object' || input === null) return false;
|
|
19
|
+
try {
|
|
20
|
+
JSON.stringify(input);
|
|
21
|
+
return true;
|
|
22
|
+
} catch { return false; }
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export function isFile(input) {
|
|
26
|
+
if (typeof input !== 'string') return false;
|
|
27
|
+
try { return fs.statSync(input).isFile(); }
|
|
28
|
+
catch { return false; }
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export function isBuffer(input) {
|
|
32
|
+
return Buffer.isBuffer(input);
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export function isStream(input) {
|
|
36
|
+
return input && typeof input.pipe === 'function';
|
|
37
|
+
}
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
import { fileTypeFromBuffer, fileTypeFromFile } from 'file-type';
|
|
2
|
+
import { open } from 'fs/promises';
|
|
3
|
+
import path from 'path';
|
|
4
|
+
|
|
5
|
+
// Extension-based MIME types for common text/code files (magic bytes don't work for these)
|
|
6
|
+
const TEXT_MIME_TYPES = {
|
|
7
|
+
'.txt': 'text/plain',
|
|
8
|
+
'.md': 'text/markdown',
|
|
9
|
+
'.json': 'application/json',
|
|
10
|
+
'.xml': 'application/xml',
|
|
11
|
+
'.html': 'text/html',
|
|
12
|
+
'.htm': 'text/html',
|
|
13
|
+
'.css': 'text/css',
|
|
14
|
+
'.js': 'text/javascript',
|
|
15
|
+
'.mjs': 'text/javascript',
|
|
16
|
+
'.ts': 'text/typescript',
|
|
17
|
+
'.tsx': 'text/typescript',
|
|
18
|
+
'.jsx': 'text/javascript',
|
|
19
|
+
'.yaml': 'text/yaml',
|
|
20
|
+
'.yml': 'text/yaml',
|
|
21
|
+
'.csv': 'text/csv',
|
|
22
|
+
'.log': 'text/plain',
|
|
23
|
+
'.sh': 'text/x-shellscript',
|
|
24
|
+
'.bash': 'text/x-shellscript',
|
|
25
|
+
'.zsh': 'text/x-shellscript',
|
|
26
|
+
'.py': 'text/x-python',
|
|
27
|
+
'.rb': 'text/x-ruby',
|
|
28
|
+
'.go': 'text/x-go',
|
|
29
|
+
'.rs': 'text/x-rust',
|
|
30
|
+
'.c': 'text/x-c',
|
|
31
|
+
'.h': 'text/x-c',
|
|
32
|
+
'.cpp': 'text/x-c++',
|
|
33
|
+
'.hpp': 'text/x-c++',
|
|
34
|
+
'.java': 'text/x-java',
|
|
35
|
+
'.sql': 'text/x-sql',
|
|
36
|
+
'.ini': 'text/plain',
|
|
37
|
+
'.conf': 'text/plain',
|
|
38
|
+
'.cfg': 'text/plain',
|
|
39
|
+
'.env': 'text/plain',
|
|
40
|
+
'.toml': 'text/toml',
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
// Heuristic: is this buffer plausibly UTF-8 text? No NUL bytes and >=95%
|
|
44
|
+
// printable / whitespace. Cheap fallback for files without magic bytes or a
|
|
45
|
+
// known extension (e.g. .ips, .rules, random log files).
|
|
46
|
+
function looksLikeText(buf) {
|
|
47
|
+
if (!buf || buf.length === 0) return false;
|
|
48
|
+
let printable = 0;
|
|
49
|
+
for (let i = 0; i < buf.length; i++) {
|
|
50
|
+
const b = buf[i];
|
|
51
|
+
if (b === 0) return false;
|
|
52
|
+
if (b === 9 || b === 10 || b === 13 || (b >= 32 && b < 127) || b >= 128) printable++;
|
|
53
|
+
}
|
|
54
|
+
return printable / buf.length >= 0.95;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
async function sniffHead(input, bytes = 4096) {
|
|
58
|
+
if (Buffer.isBuffer(input)) return input.subarray(0, bytes);
|
|
59
|
+
const fh = await open(input, 'r');
|
|
60
|
+
try {
|
|
61
|
+
const out = Buffer.alloc(bytes);
|
|
62
|
+
const { bytesRead } = await fh.read(out, 0, bytes, 0);
|
|
63
|
+
return out.subarray(0, bytesRead);
|
|
64
|
+
} finally { await fh.close(); }
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
// Streaming variant: detect from an already-captured head buffer (first few KB)
|
|
68
|
+
// plus the storage key's extension. Used by the streaming put path where the
|
|
69
|
+
// full blob is never materialized.
|
|
70
|
+
export async function detectMimeFromHead(head, key) {
|
|
71
|
+
try {
|
|
72
|
+
if (head && head.length) {
|
|
73
|
+
const result = await fileTypeFromBuffer(head);
|
|
74
|
+
if (result?.mime) return result.mime;
|
|
75
|
+
}
|
|
76
|
+
if (typeof key === 'string') {
|
|
77
|
+
const ext = path.extname(key).toLowerCase();
|
|
78
|
+
if (TEXT_MIME_TYPES[ext]) return TEXT_MIME_TYPES[ext];
|
|
79
|
+
}
|
|
80
|
+
if (head && looksLikeText(head)) return 'text/plain';
|
|
81
|
+
return 'application/octet-stream';
|
|
82
|
+
} catch {
|
|
83
|
+
return 'application/octet-stream';
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
export async function detectMimeType(input) {
|
|
88
|
+
try {
|
|
89
|
+
const result = Buffer.isBuffer(input)
|
|
90
|
+
? await fileTypeFromBuffer(input)
|
|
91
|
+
: await fileTypeFromFile(input);
|
|
92
|
+
if (result?.mime) return result.mime;
|
|
93
|
+
|
|
94
|
+
if (typeof input === 'string') {
|
|
95
|
+
const ext = path.extname(input).toLowerCase();
|
|
96
|
+
if (TEXT_MIME_TYPES[ext]) return TEXT_MIME_TYPES[ext];
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
const head = await sniffHead(input).catch(() => null);
|
|
100
|
+
if (head && looksLikeText(head)) return 'text/plain';
|
|
101
|
+
|
|
102
|
+
return 'application/octet-stream';
|
|
103
|
+
} catch {
|
|
104
|
+
return 'application/octet-stream';
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
import fs from 'fs';
|
|
2
|
+
import path from 'path';
|
|
3
|
+
import Debug from 'debug';
|
|
4
|
+
|
|
5
|
+
const debug = Debug('stored:mount');
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Filesystem types served over the network. A backend rooted on one of these is
|
|
9
|
+
* `remote: true` — the bytes are reachable through a POSIX path, but every
|
|
10
|
+
* assumption that makes local disk cheap (inotify, free stat, fast full-file
|
|
11
|
+
* hashing) is off.
|
|
12
|
+
*
|
|
13
|
+
* `fuse.*` is matched by prefix: sshfs, rclone, s3fs and friends are all remote
|
|
14
|
+
* regardless of the transport they wrap.
|
|
15
|
+
*/
|
|
16
|
+
const REMOTE_FSTYPES = new Set([
|
|
17
|
+
'cifs', 'smbfs', 'smb3',
|
|
18
|
+
'nfs', 'nfs4',
|
|
19
|
+
'afpfs', 'afp',
|
|
20
|
+
'ncpfs', 'coda',
|
|
21
|
+
'9p', 'ceph', 'glusterfs', 'lustre', 'beegfs', 'ocfs2', 'gfs2',
|
|
22
|
+
'davfs', 'davfs2', 'webdav',
|
|
23
|
+
]);
|
|
24
|
+
|
|
25
|
+
const REMOTE_FSTYPE_PREFIXES = ['fuse.'];
|
|
26
|
+
|
|
27
|
+
/** `\040`-style octal escapes are how /proc/mounts encodes spaces and tabs. */
|
|
28
|
+
function unescapeMountField(value) {
|
|
29
|
+
return String(value).replace(/\\(\d{3})/g, (_, oct) => String.fromCharCode(parseInt(oct, 8)));
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export function isRemoteFstype(fstype) {
|
|
33
|
+
if (!fstype) return false;
|
|
34
|
+
const type = String(fstype).toLowerCase();
|
|
35
|
+
if (REMOTE_FSTYPES.has(type)) return true;
|
|
36
|
+
return REMOTE_FSTYPE_PREFIXES.some((prefix) => type.startsWith(prefix));
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Read the kernel mount table. Linux only — every other platform returns [] and
|
|
41
|
+
* detection degrades to "assume local unless configured otherwise".
|
|
42
|
+
* @returns {Array<{ source: string, mountpoint: string, fstype: string }>}
|
|
43
|
+
*/
|
|
44
|
+
function readMountTable() {
|
|
45
|
+
if (process.platform !== 'linux') return [];
|
|
46
|
+
let raw;
|
|
47
|
+
try {
|
|
48
|
+
raw = fs.readFileSync('/proc/mounts', 'utf8');
|
|
49
|
+
} catch (err) {
|
|
50
|
+
debug(`Cannot read /proc/mounts: ${err.message}`);
|
|
51
|
+
return [];
|
|
52
|
+
}
|
|
53
|
+
const rows = [];
|
|
54
|
+
for (const line of raw.split('\n')) {
|
|
55
|
+
if (!line) continue;
|
|
56
|
+
const [source, mountpoint, fstype] = line.split(' ');
|
|
57
|
+
if (!mountpoint || !fstype) continue;
|
|
58
|
+
rows.push({
|
|
59
|
+
source: unescapeMountField(source),
|
|
60
|
+
mountpoint: unescapeMountField(mountpoint),
|
|
61
|
+
fstype: fstype.toLowerCase(),
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
return rows;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Which mount serves `absPath`? The longest mountpoint that is a path-prefix of
|
|
69
|
+
* it wins (a bind/overlay nested inside another mount must beat its parent).
|
|
70
|
+
*
|
|
71
|
+
* Deliberately synchronous and dependency-free: this runs in the FileBackend
|
|
72
|
+
* constructor, where a subprocess (`findmnt`) or an async hop would force every
|
|
73
|
+
* caller of `addBackend()` to become async. Reading /proc/mounts is a single
|
|
74
|
+
* cheap read of a kernel-generated file.
|
|
75
|
+
*
|
|
76
|
+
* @param {string} absPath Absolute, ideally already-realpath'd directory.
|
|
77
|
+
* @returns {{ remote: boolean, transport: string|null, source: string|null, mountpoint: string|null }}
|
|
78
|
+
*/
|
|
79
|
+
export function detectMountSync(absPath) {
|
|
80
|
+
const unknown = { remote: false, transport: null, source: null, mountpoint: null };
|
|
81
|
+
if (!absPath || typeof absPath !== 'string') return unknown;
|
|
82
|
+
|
|
83
|
+
const target = path.resolve(absPath);
|
|
84
|
+
const table = readMountTable();
|
|
85
|
+
if (!table.length) return unknown;
|
|
86
|
+
|
|
87
|
+
let best = null;
|
|
88
|
+
for (const row of table) {
|
|
89
|
+
const mp = row.mountpoint;
|
|
90
|
+
const covers = mp === '/' ? true : (target === mp || target.startsWith(mp + path.sep));
|
|
91
|
+
if (!covers) continue;
|
|
92
|
+
if (!best || mp.length > best.mountpoint.length) best = row;
|
|
93
|
+
}
|
|
94
|
+
if (!best) return unknown;
|
|
95
|
+
|
|
96
|
+
const remote = isRemoteFstype(best.fstype);
|
|
97
|
+
debug(`${target} → ${best.fstype} on ${best.mountpoint} (${remote ? 'remote' : 'local'})`);
|
|
98
|
+
return { remote, transport: best.fstype, source: best.source, mountpoint: best.mountpoint };
|
|
99
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,51 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@augmentd-labs/canvas-edge",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "0.4.1",
|
|
4
|
+
"description": "Canvas device runtime: folder mirrors (canvas-stored Mirror engine) + edge tunnel client (dist artifact)",
|
|
5
|
+
"license": "AGPL-3.0-or-later",
|
|
6
|
+
"type": "module",
|
|
7
|
+
"bin": {
|
|
8
|
+
"canvas-edge": "bin/canvas-edge"
|
|
9
|
+
},
|
|
10
|
+
"exports": {
|
|
11
|
+
".": "./src/index.js",
|
|
12
|
+
"./client": "./src/EdgeClient.js"
|
|
13
|
+
},
|
|
14
|
+
"files": [
|
|
15
|
+
"bin",
|
|
16
|
+
"src",
|
|
17
|
+
"NOTICE"
|
|
18
|
+
],
|
|
19
|
+
"engines": {
|
|
20
|
+
"node": ">=20.18.0"
|
|
21
|
+
},
|
|
22
|
+
"dependencies": {
|
|
23
|
+
"@augmentd-labs/canvas-protocol": "^0.4.0",
|
|
24
|
+
"canvas-stored": "1.9.1",
|
|
25
|
+
"cacache": "^20.0.4",
|
|
26
|
+
"chokidar": "^5.0.0",
|
|
27
|
+
"debug": "^4.4.3",
|
|
28
|
+
"eventemitter2": "^6.4.9",
|
|
29
|
+
"exifr": "^7.1.3",
|
|
30
|
+
"file-type": "^21.3.3",
|
|
31
|
+
"fs-extra": "^11.3.4",
|
|
32
|
+
"image-size": "^2.0.2",
|
|
33
|
+
"imap": "^0.8.19",
|
|
34
|
+
"lmdb": "^3.5.2",
|
|
35
|
+
"music-metadata": "^11.9.0",
|
|
36
|
+
"picomatch": "^2.3.1",
|
|
37
|
+
"fastify": "^4.29.1",
|
|
38
|
+
"pino": "^10.3.1",
|
|
39
|
+
"socket.io-client": "^4.8.3"
|
|
40
|
+
},
|
|
41
|
+
"repository": {
|
|
42
|
+
"type": "git",
|
|
43
|
+
"url": "git+https://github.com/canvas-ui/canvas-common.git",
|
|
44
|
+
"directory": "runtimes/edge"
|
|
45
|
+
},
|
|
46
|
+
"bundleDependencies": [
|
|
47
|
+
"canvas-stored"
|
|
48
|
+
],
|
|
49
|
+
"canvasRev": "0dbc541",
|
|
50
|
+
"canvasSource": "runtimes/edge"
|
|
51
|
+
}
|