hazo_files 1.6.0 → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGE_LOG.md +24 -0
- package/README.md +161 -0
- package/dist/server/index.d.mts +128 -1
- package/dist/server/index.d.ts +128 -1
- package/dist/server/index.js +296 -0
- package/dist/server/index.mjs +297 -0
- package/dist/testing/index.d.mts +62 -0
- package/dist/testing/index.d.ts +62 -0
- package/dist/testing/index.js +71 -0
- package/dist/testing/index.mjs +44 -0
- package/docs/superpowers/plans/2026-05-23-test-app-v2-providers.md +968 -0
- package/package.json +12 -4
package/dist/server/index.js
CHANGED
|
@@ -31,6 +31,7 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
|
|
|
31
31
|
var index_exports = {};
|
|
32
32
|
__export(index_exports, {
|
|
33
33
|
ALL_SYSTEM_VARIABLES: () => ALL_SYSTEM_VARIABLES,
|
|
34
|
+
AppFileServerProvider: () => AppFileServerProvider,
|
|
34
35
|
AuthenticationError: () => AuthenticationError,
|
|
35
36
|
ConfigurationError: () => ConfigurationError,
|
|
36
37
|
DEFAULT_DATE_FORMATS: () => DEFAULT_DATE_FORMATS,
|
|
@@ -46,6 +47,7 @@ __export(index_exports, {
|
|
|
46
47
|
FileTooLargeError: () => FileTooLargeError,
|
|
47
48
|
GoogleDriveAuth: () => GoogleDriveAuth,
|
|
48
49
|
GoogleDriveModule: () => GoogleDriveModule,
|
|
50
|
+
GoogleDriveProvider: () => GoogleDriveProvider,
|
|
49
51
|
HAZO_FILES_DEFAULT_TABLE_NAME: () => HAZO_FILES_DEFAULT_TABLE_NAME,
|
|
50
52
|
HAZO_FILES_JOB_TYPES: () => HAZO_FILES_JOB_TYPES,
|
|
51
53
|
HAZO_FILES_MIGRATION_V2: () => HAZO_FILES_MIGRATION_V2,
|
|
@@ -58,6 +60,7 @@ __export(index_exports, {
|
|
|
58
60
|
HAZO_FILE_QUOTAS_TABLE_SCHEMA: () => HAZO_FILE_QUOTAS_TABLE_SCHEMA,
|
|
59
61
|
HazoFilesError: () => HazoFilesError,
|
|
60
62
|
ImportSizeCapError: () => ImportSizeCapError,
|
|
63
|
+
InMemoryProvider: () => InMemoryProvider,
|
|
61
64
|
InvalidExtensionError: () => InvalidExtensionError,
|
|
62
65
|
InvalidPathError: () => InvalidPathError,
|
|
63
66
|
LLMExtractionService: () => LLMExtractionService,
|
|
@@ -71,6 +74,9 @@ __export(index_exports, {
|
|
|
71
74
|
SYSTEM_COUNTER_VARIABLES: () => SYSTEM_COUNTER_VARIABLES,
|
|
72
75
|
SYSTEM_DATE_VARIABLES: () => SYSTEM_DATE_VARIABLES,
|
|
73
76
|
SYSTEM_FILE_VARIABLES: () => SYSTEM_FILE_VARIABLES,
|
|
77
|
+
StorageCollisionExhausted: () => StorageCollisionExhausted,
|
|
78
|
+
StorageNotConfigured: () => StorageNotConfigured,
|
|
79
|
+
StorageUnavailable: () => StorageUnavailable,
|
|
74
80
|
TrackedFileManager: () => TrackedFileManager,
|
|
75
81
|
UploadExtractService: () => UploadExtractService,
|
|
76
82
|
addExtractionToFileData: () => addExtractionToFileData,
|
|
@@ -6477,6 +6483,290 @@ var HAZO_FILE_QUOTAS_TABLE_SCHEMA = {
|
|
|
6477
6483
|
columns: ["scope_id", "byte_limit", "byte_used", "updated_at"]
|
|
6478
6484
|
};
|
|
6479
6485
|
|
|
6486
|
+
// src/providers/app-file-server.ts
|
|
6487
|
+
var import_node_crypto = require("crypto");
|
|
6488
|
+
var import_node_fs = require("fs");
|
|
6489
|
+
var import_node_path = require("path");
|
|
6490
|
+
var AppFileServerProvider = class {
|
|
6491
|
+
constructor(opts) {
|
|
6492
|
+
this.provider_tag = "app_file_server";
|
|
6493
|
+
this.root = (0, import_node_path.normalize)(opts.root);
|
|
6494
|
+
this.secret = opts.hmac_secret;
|
|
6495
|
+
this.default_ttl = opts.default_ttl_seconds ?? 300;
|
|
6496
|
+
}
|
|
6497
|
+
resolve(path4) {
|
|
6498
|
+
const safe = (0, import_node_path.normalize)(path4).replace(/^([./\\])+/, "");
|
|
6499
|
+
if (safe.includes(`..${import_node_path.sep}`) || safe.startsWith("..")) {
|
|
6500
|
+
throw new Error(`Path escapes root: ${path4}`);
|
|
6501
|
+
}
|
|
6502
|
+
return (0, import_node_path.join)(this.root, safe);
|
|
6503
|
+
}
|
|
6504
|
+
async put(path4, body, opts) {
|
|
6505
|
+
const abs = this.resolve(path4);
|
|
6506
|
+
if (opts?.ifNotExists && (0, import_node_fs.existsSync)(abs)) {
|
|
6507
|
+
throw new Error(`File exists: ${path4}`);
|
|
6508
|
+
}
|
|
6509
|
+
(0, import_node_fs.mkdirSync)((0, import_node_path.dirname)(abs), { recursive: true });
|
|
6510
|
+
const buf = Buffer.isBuffer(body) ? body : await streamToBuffer(body);
|
|
6511
|
+
(0, import_node_fs.writeFileSync)(abs, buf);
|
|
6512
|
+
return { provider: this.provider_tag, native_id: path4, size: buf.length };
|
|
6513
|
+
}
|
|
6514
|
+
async get(path4) {
|
|
6515
|
+
return (0, import_node_fs.readFileSync)(this.resolve(path4));
|
|
6516
|
+
}
|
|
6517
|
+
async delete(path4) {
|
|
6518
|
+
const abs = this.resolve(path4);
|
|
6519
|
+
if ((0, import_node_fs.existsSync)(abs)) (0, import_node_fs.rmSync)(abs, { force: true });
|
|
6520
|
+
}
|
|
6521
|
+
async exists(path4) {
|
|
6522
|
+
return (0, import_node_fs.existsSync)(this.resolve(path4));
|
|
6523
|
+
}
|
|
6524
|
+
async getSignedUrl(path4, opts) {
|
|
6525
|
+
const ttl = opts?.ttl_seconds ?? this.default_ttl;
|
|
6526
|
+
const exp = Math.floor(Date.now() / 1e3) + ttl;
|
|
6527
|
+
const payload = `${path4}:${exp}`;
|
|
6528
|
+
const sig = (0, import_node_crypto.createHmac)("sha256", this.secret).update(payload).digest("base64url");
|
|
6529
|
+
const token = `${exp}.${sig}`;
|
|
6530
|
+
return `/api/files/serve/${token}/${path4}`;
|
|
6531
|
+
}
|
|
6532
|
+
/** Verify a token produced by `getSignedUrl`. Used by the `/api/files/serve` route. */
|
|
6533
|
+
verifySignedUrl(token, path4) {
|
|
6534
|
+
const [expStr, sig] = token.split(".");
|
|
6535
|
+
const exp = Number(expStr);
|
|
6536
|
+
if (!Number.isFinite(exp) || exp * 1e3 < Date.now()) return false;
|
|
6537
|
+
const expected = (0, import_node_crypto.createHmac)("sha256", this.secret).update(`${path4}:${exp}`).digest("base64url");
|
|
6538
|
+
return sig === expected;
|
|
6539
|
+
}
|
|
6540
|
+
async probe() {
|
|
6541
|
+
try {
|
|
6542
|
+
const test = (0, import_node_path.join)(this.root, ".hazo_probe");
|
|
6543
|
+
(0, import_node_fs.mkdirSync)(this.root, { recursive: true });
|
|
6544
|
+
(0, import_node_fs.writeFileSync)(test, "probe");
|
|
6545
|
+
(0, import_node_fs.statSync)(test);
|
|
6546
|
+
(0, import_node_fs.rmSync)(test, { force: true });
|
|
6547
|
+
return { ok: true };
|
|
6548
|
+
} catch (err) {
|
|
6549
|
+
return { ok: false, error: "write_denied", message: String(err) };
|
|
6550
|
+
}
|
|
6551
|
+
}
|
|
6552
|
+
};
|
|
6553
|
+
async function streamToBuffer(s) {
|
|
6554
|
+
const chunks = [];
|
|
6555
|
+
for await (const c of s) chunks.push(Buffer.isBuffer(c) ? c : Buffer.from(c));
|
|
6556
|
+
return Buffer.concat(chunks);
|
|
6557
|
+
}
|
|
6558
|
+
|
|
6559
|
+
// src/providers/in-memory.ts
|
|
6560
|
+
var InMemoryProvider = class {
|
|
6561
|
+
constructor() {
|
|
6562
|
+
this.provider_tag = "in_memory";
|
|
6563
|
+
this.store = /* @__PURE__ */ new Map();
|
|
6564
|
+
}
|
|
6565
|
+
async put(path4, body, opts) {
|
|
6566
|
+
if (opts?.ifNotExists && this.store.has(path4)) throw new Error(`File exists: ${path4}`);
|
|
6567
|
+
const buf = Buffer.isBuffer(body) ? body : await streamToBuffer2(body);
|
|
6568
|
+
this.store.set(path4, buf);
|
|
6569
|
+
return { provider: this.provider_tag, native_id: path4, size: buf.length };
|
|
6570
|
+
}
|
|
6571
|
+
async get(path4) {
|
|
6572
|
+
const buf = this.store.get(path4);
|
|
6573
|
+
if (!buf) throw new Error(`Not found: ${path4}`);
|
|
6574
|
+
return buf;
|
|
6575
|
+
}
|
|
6576
|
+
async delete(path4) {
|
|
6577
|
+
this.store.delete(path4);
|
|
6578
|
+
}
|
|
6579
|
+
async exists(path4) {
|
|
6580
|
+
return this.store.has(path4);
|
|
6581
|
+
}
|
|
6582
|
+
async getSignedUrl(path4, _opts) {
|
|
6583
|
+
const buf = this.store.get(path4);
|
|
6584
|
+
if (!buf) throw new Error(`Not found: ${path4}`);
|
|
6585
|
+
return `data:application/octet-stream;base64,${buf.toString("base64")}`;
|
|
6586
|
+
}
|
|
6587
|
+
async probe() {
|
|
6588
|
+
return { ok: true };
|
|
6589
|
+
}
|
|
6590
|
+
/** Test-only escape hatch — exposes the internal store for assertions. */
|
|
6591
|
+
snapshot() {
|
|
6592
|
+
return new Map(this.store);
|
|
6593
|
+
}
|
|
6594
|
+
};
|
|
6595
|
+
async function streamToBuffer2(s) {
|
|
6596
|
+
const chunks = [];
|
|
6597
|
+
for await (const c of s) chunks.push(Buffer.isBuffer(c) ? c : Buffer.from(c));
|
|
6598
|
+
return Buffer.concat(chunks);
|
|
6599
|
+
}
|
|
6600
|
+
|
|
6601
|
+
// src/providers/google-drive.ts
|
|
6602
|
+
var import_googleapis3 = require("googleapis");
|
|
6603
|
+
var GoogleDriveProvider = class {
|
|
6604
|
+
constructor(opts) {
|
|
6605
|
+
this.provider_tag = "gdrive";
|
|
6606
|
+
let credentials = {};
|
|
6607
|
+
try {
|
|
6608
|
+
credentials = JSON.parse(opts.service_account_json);
|
|
6609
|
+
} catch {
|
|
6610
|
+
}
|
|
6611
|
+
const auth = new import_googleapis3.google.auth.GoogleAuth({
|
|
6612
|
+
credentials,
|
|
6613
|
+
scopes: ["https://www.googleapis.com/auth/drive"]
|
|
6614
|
+
});
|
|
6615
|
+
this.drive = import_googleapis3.google.drive({ version: "v3", auth });
|
|
6616
|
+
this.driveId = opts.shared_drive_id;
|
|
6617
|
+
this.cache = opts.path_cache;
|
|
6618
|
+
}
|
|
6619
|
+
async probe() {
|
|
6620
|
+
try {
|
|
6621
|
+
await this.drive.drives.get({
|
|
6622
|
+
driveId: this.driveId,
|
|
6623
|
+
supportsAllDrives: true
|
|
6624
|
+
});
|
|
6625
|
+
return { ok: true };
|
|
6626
|
+
} catch (err) {
|
|
6627
|
+
const e = err;
|
|
6628
|
+
if (e?.code === 404) return { ok: false, error: "drive_not_shared", message: "SA cannot see Shared Drive" };
|
|
6629
|
+
if (e?.code === 403) return { ok: false, error: "write_denied", message: "SA lacks permission" };
|
|
6630
|
+
return { ok: false, error: "transient", message: String(err) };
|
|
6631
|
+
}
|
|
6632
|
+
}
|
|
6633
|
+
async put(path4, body, opts) {
|
|
6634
|
+
const segments = path4.split("/").filter(Boolean);
|
|
6635
|
+
const filename = segments.pop();
|
|
6636
|
+
const parentId = await this.resolvePath(segments, { create: true });
|
|
6637
|
+
if (opts?.ifNotExists) {
|
|
6638
|
+
const existing = await this.findChild(filename, parentId);
|
|
6639
|
+
if (existing) throw new Error(`File exists: ${path4}`);
|
|
6640
|
+
}
|
|
6641
|
+
const buf = Buffer.isBuffer(body) ? body : await streamToBuffer3(body);
|
|
6642
|
+
const res = await this.drive.files.create({
|
|
6643
|
+
requestBody: {
|
|
6644
|
+
name: filename,
|
|
6645
|
+
parents: [parentId],
|
|
6646
|
+
mimeType: opts?.contentType ?? "application/octet-stream"
|
|
6647
|
+
},
|
|
6648
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
6649
|
+
media: { body: buf },
|
|
6650
|
+
fields: "id, name"
|
|
6651
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
6652
|
+
});
|
|
6653
|
+
return { provider: this.provider_tag, native_id: res.data.id, size: buf.length };
|
|
6654
|
+
}
|
|
6655
|
+
async get(path4) {
|
|
6656
|
+
const id = await this.lookupFileId(path4);
|
|
6657
|
+
const res = await this.drive.files.get(
|
|
6658
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
6659
|
+
{ fileId: id, alt: "media", supportsAllDrives: true },
|
|
6660
|
+
{ responseType: "arraybuffer" }
|
|
6661
|
+
);
|
|
6662
|
+
return Buffer.from(res.data);
|
|
6663
|
+
}
|
|
6664
|
+
async delete(path4) {
|
|
6665
|
+
try {
|
|
6666
|
+
const id = await this.lookupFileId(path4);
|
|
6667
|
+
await this.drive.files.delete({ fileId: id, supportsAllDrives: true });
|
|
6668
|
+
} catch (err) {
|
|
6669
|
+
const e = err;
|
|
6670
|
+
if (e?.code !== 404) throw err;
|
|
6671
|
+
}
|
|
6672
|
+
}
|
|
6673
|
+
async exists(path4) {
|
|
6674
|
+
try {
|
|
6675
|
+
await this.lookupFileId(path4);
|
|
6676
|
+
return true;
|
|
6677
|
+
} catch {
|
|
6678
|
+
return false;
|
|
6679
|
+
}
|
|
6680
|
+
}
|
|
6681
|
+
async getSignedUrl(path4, _opts) {
|
|
6682
|
+
const id = await this.lookupFileId(path4);
|
|
6683
|
+
return `https://drive.google.com/uc?id=${id}&export=download`;
|
|
6684
|
+
}
|
|
6685
|
+
async resolvePath(segments, opts) {
|
|
6686
|
+
let parentId = this.driveId;
|
|
6687
|
+
const accumulated = [];
|
|
6688
|
+
for (const segment of segments) {
|
|
6689
|
+
accumulated.push(segment);
|
|
6690
|
+
const cacheKey = accumulated.join("/");
|
|
6691
|
+
const cached = await this.cache.lookup(cacheKey);
|
|
6692
|
+
if (cached) {
|
|
6693
|
+
parentId = cached;
|
|
6694
|
+
continue;
|
|
6695
|
+
}
|
|
6696
|
+
let found = await this.findChild(segment, parentId, "application/vnd.google-apps.folder");
|
|
6697
|
+
if (!found && opts.create) {
|
|
6698
|
+
const created = await this.drive.files.create({
|
|
6699
|
+
requestBody: {
|
|
6700
|
+
name: segment,
|
|
6701
|
+
parents: [parentId],
|
|
6702
|
+
mimeType: "application/vnd.google-apps.folder"
|
|
6703
|
+
},
|
|
6704
|
+
fields: "id"
|
|
6705
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
6706
|
+
});
|
|
6707
|
+
found = created.data.id;
|
|
6708
|
+
}
|
|
6709
|
+
if (!found) throw new Error(`Folder not found and create=false: ${cacheKey}`);
|
|
6710
|
+
await this.cache.write(cacheKey, found);
|
|
6711
|
+
parentId = found;
|
|
6712
|
+
}
|
|
6713
|
+
return parentId;
|
|
6714
|
+
}
|
|
6715
|
+
async findChild(name, parentId, mimeType) {
|
|
6716
|
+
const q = [
|
|
6717
|
+
`name='${name.replace(/'/g, "\\'")}'`,
|
|
6718
|
+
`'${parentId}' in parents`,
|
|
6719
|
+
`trashed=false`
|
|
6720
|
+
];
|
|
6721
|
+
if (mimeType) q.push(`mimeType='${mimeType}'`);
|
|
6722
|
+
const res = await this.drive.files.list({
|
|
6723
|
+
q: q.join(" and "),
|
|
6724
|
+
driveId: this.driveId,
|
|
6725
|
+
corpora: "drive",
|
|
6726
|
+
includeItemsFromAllDrives: true,
|
|
6727
|
+
fields: "files(id, name)"
|
|
6728
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
6729
|
+
});
|
|
6730
|
+
return res.data.files?.[0]?.id ?? null;
|
|
6731
|
+
}
|
|
6732
|
+
async lookupFileId(path4) {
|
|
6733
|
+
const segments = path4.split("/").filter(Boolean);
|
|
6734
|
+
const filename = segments.pop();
|
|
6735
|
+
const parentId = await this.resolvePath(segments, { create: false });
|
|
6736
|
+
const id = await this.findChild(filename, parentId);
|
|
6737
|
+
if (!id) throw Object.assign(new Error(`Not found: ${path4}`), { code: 404 });
|
|
6738
|
+
return id;
|
|
6739
|
+
}
|
|
6740
|
+
};
|
|
6741
|
+
async function streamToBuffer3(s) {
|
|
6742
|
+
const chunks = [];
|
|
6743
|
+
for await (const c of s) chunks.push(Buffer.isBuffer(c) ? c : Buffer.from(c));
|
|
6744
|
+
return Buffer.concat(chunks);
|
|
6745
|
+
}
|
|
6746
|
+
|
|
6747
|
+
// src/providers/types.ts
|
|
6748
|
+
var StorageCollisionExhausted = class extends Error {
|
|
6749
|
+
constructor(attempts, lastPath) {
|
|
6750
|
+
super(`Storage collision could not be resolved after ${attempts} attempts at "${lastPath}"`);
|
|
6751
|
+
this.attempts = attempts;
|
|
6752
|
+
this.lastPath = lastPath;
|
|
6753
|
+
this.name = "StorageCollisionExhausted";
|
|
6754
|
+
}
|
|
6755
|
+
};
|
|
6756
|
+
var StorageNotConfigured = class extends Error {
|
|
6757
|
+
constructor() {
|
|
6758
|
+
super("Storage provider is not configured for this scope");
|
|
6759
|
+
this.name = "StorageNotConfigured";
|
|
6760
|
+
}
|
|
6761
|
+
};
|
|
6762
|
+
var StorageUnavailable = class extends Error {
|
|
6763
|
+
constructor(reason, message) {
|
|
6764
|
+
super(message);
|
|
6765
|
+
this.reason = reason;
|
|
6766
|
+
this.name = "StorageUnavailable";
|
|
6767
|
+
}
|
|
6768
|
+
};
|
|
6769
|
+
|
|
6480
6770
|
// src/migrations/add-reference-tracking.ts
|
|
6481
6771
|
async function migrateToV2(executor, dbType, tableName) {
|
|
6482
6772
|
const migration = tableName ? getMigrationForTable(tableName, dbType) : HAZO_FILES_MIGRATION_V2[dbType];
|
|
@@ -6520,6 +6810,7 @@ try {
|
|
|
6520
6810
|
// Annotate the CommonJS export names for ESM import in node:
|
|
6521
6811
|
0 && (module.exports = {
|
|
6522
6812
|
ALL_SYSTEM_VARIABLES,
|
|
6813
|
+
AppFileServerProvider,
|
|
6523
6814
|
AuthenticationError,
|
|
6524
6815
|
ConfigurationError,
|
|
6525
6816
|
DEFAULT_DATE_FORMATS,
|
|
@@ -6535,6 +6826,7 @@ try {
|
|
|
6535
6826
|
FileTooLargeError,
|
|
6536
6827
|
GoogleDriveAuth,
|
|
6537
6828
|
GoogleDriveModule,
|
|
6829
|
+
GoogleDriveProvider,
|
|
6538
6830
|
HAZO_FILES_DEFAULT_TABLE_NAME,
|
|
6539
6831
|
HAZO_FILES_JOB_TYPES,
|
|
6540
6832
|
HAZO_FILES_MIGRATION_V2,
|
|
@@ -6547,6 +6839,7 @@ try {
|
|
|
6547
6839
|
HAZO_FILE_QUOTAS_TABLE_SCHEMA,
|
|
6548
6840
|
HazoFilesError,
|
|
6549
6841
|
ImportSizeCapError,
|
|
6842
|
+
InMemoryProvider,
|
|
6550
6843
|
InvalidExtensionError,
|
|
6551
6844
|
InvalidPathError,
|
|
6552
6845
|
LLMExtractionService,
|
|
@@ -6560,6 +6853,9 @@ try {
|
|
|
6560
6853
|
SYSTEM_COUNTER_VARIABLES,
|
|
6561
6854
|
SYSTEM_DATE_VARIABLES,
|
|
6562
6855
|
SYSTEM_FILE_VARIABLES,
|
|
6856
|
+
StorageCollisionExhausted,
|
|
6857
|
+
StorageNotConfigured,
|
|
6858
|
+
StorageUnavailable,
|
|
6563
6859
|
TrackedFileManager,
|
|
6564
6860
|
UploadExtractService,
|
|
6565
6861
|
addExtractionToFileData,
|
package/dist/server/index.mjs
CHANGED
|
@@ -6288,6 +6288,297 @@ var HAZO_FILE_QUOTAS_TABLE_SCHEMA = {
|
|
|
6288
6288
|
columns: ["scope_id", "byte_limit", "byte_used", "updated_at"]
|
|
6289
6289
|
};
|
|
6290
6290
|
|
|
6291
|
+
// src/providers/app-file-server.ts
|
|
6292
|
+
import { createHmac } from "crypto";
|
|
6293
|
+
import {
|
|
6294
|
+
existsSync as existsSync2,
|
|
6295
|
+
mkdirSync,
|
|
6296
|
+
readFileSync as readFileSync2,
|
|
6297
|
+
rmSync,
|
|
6298
|
+
statSync,
|
|
6299
|
+
writeFileSync
|
|
6300
|
+
} from "fs";
|
|
6301
|
+
import { dirname as dirname2, join as join4, normalize, sep } from "path";
|
|
6302
|
+
var AppFileServerProvider = class {
|
|
6303
|
+
constructor(opts) {
|
|
6304
|
+
this.provider_tag = "app_file_server";
|
|
6305
|
+
this.root = normalize(opts.root);
|
|
6306
|
+
this.secret = opts.hmac_secret;
|
|
6307
|
+
this.default_ttl = opts.default_ttl_seconds ?? 300;
|
|
6308
|
+
}
|
|
6309
|
+
resolve(path4) {
|
|
6310
|
+
const safe = normalize(path4).replace(/^([./\\])+/, "");
|
|
6311
|
+
if (safe.includes(`..${sep}`) || safe.startsWith("..")) {
|
|
6312
|
+
throw new Error(`Path escapes root: ${path4}`);
|
|
6313
|
+
}
|
|
6314
|
+
return join4(this.root, safe);
|
|
6315
|
+
}
|
|
6316
|
+
async put(path4, body, opts) {
|
|
6317
|
+
const abs = this.resolve(path4);
|
|
6318
|
+
if (opts?.ifNotExists && existsSync2(abs)) {
|
|
6319
|
+
throw new Error(`File exists: ${path4}`);
|
|
6320
|
+
}
|
|
6321
|
+
mkdirSync(dirname2(abs), { recursive: true });
|
|
6322
|
+
const buf = Buffer.isBuffer(body) ? body : await streamToBuffer(body);
|
|
6323
|
+
writeFileSync(abs, buf);
|
|
6324
|
+
return { provider: this.provider_tag, native_id: path4, size: buf.length };
|
|
6325
|
+
}
|
|
6326
|
+
async get(path4) {
|
|
6327
|
+
return readFileSync2(this.resolve(path4));
|
|
6328
|
+
}
|
|
6329
|
+
async delete(path4) {
|
|
6330
|
+
const abs = this.resolve(path4);
|
|
6331
|
+
if (existsSync2(abs)) rmSync(abs, { force: true });
|
|
6332
|
+
}
|
|
6333
|
+
async exists(path4) {
|
|
6334
|
+
return existsSync2(this.resolve(path4));
|
|
6335
|
+
}
|
|
6336
|
+
async getSignedUrl(path4, opts) {
|
|
6337
|
+
const ttl = opts?.ttl_seconds ?? this.default_ttl;
|
|
6338
|
+
const exp = Math.floor(Date.now() / 1e3) + ttl;
|
|
6339
|
+
const payload = `${path4}:${exp}`;
|
|
6340
|
+
const sig = createHmac("sha256", this.secret).update(payload).digest("base64url");
|
|
6341
|
+
const token = `${exp}.${sig}`;
|
|
6342
|
+
return `/api/files/serve/${token}/${path4}`;
|
|
6343
|
+
}
|
|
6344
|
+
/** Verify a token produced by `getSignedUrl`. Used by the `/api/files/serve` route. */
|
|
6345
|
+
verifySignedUrl(token, path4) {
|
|
6346
|
+
const [expStr, sig] = token.split(".");
|
|
6347
|
+
const exp = Number(expStr);
|
|
6348
|
+
if (!Number.isFinite(exp) || exp * 1e3 < Date.now()) return false;
|
|
6349
|
+
const expected = createHmac("sha256", this.secret).update(`${path4}:${exp}`).digest("base64url");
|
|
6350
|
+
return sig === expected;
|
|
6351
|
+
}
|
|
6352
|
+
async probe() {
|
|
6353
|
+
try {
|
|
6354
|
+
const test = join4(this.root, ".hazo_probe");
|
|
6355
|
+
mkdirSync(this.root, { recursive: true });
|
|
6356
|
+
writeFileSync(test, "probe");
|
|
6357
|
+
statSync(test);
|
|
6358
|
+
rmSync(test, { force: true });
|
|
6359
|
+
return { ok: true };
|
|
6360
|
+
} catch (err) {
|
|
6361
|
+
return { ok: false, error: "write_denied", message: String(err) };
|
|
6362
|
+
}
|
|
6363
|
+
}
|
|
6364
|
+
};
|
|
6365
|
+
async function streamToBuffer(s) {
|
|
6366
|
+
const chunks = [];
|
|
6367
|
+
for await (const c of s) chunks.push(Buffer.isBuffer(c) ? c : Buffer.from(c));
|
|
6368
|
+
return Buffer.concat(chunks);
|
|
6369
|
+
}
|
|
6370
|
+
|
|
6371
|
+
// src/providers/in-memory.ts
|
|
6372
|
+
var InMemoryProvider = class {
|
|
6373
|
+
constructor() {
|
|
6374
|
+
this.provider_tag = "in_memory";
|
|
6375
|
+
this.store = /* @__PURE__ */ new Map();
|
|
6376
|
+
}
|
|
6377
|
+
async put(path4, body, opts) {
|
|
6378
|
+
if (opts?.ifNotExists && this.store.has(path4)) throw new Error(`File exists: ${path4}`);
|
|
6379
|
+
const buf = Buffer.isBuffer(body) ? body : await streamToBuffer2(body);
|
|
6380
|
+
this.store.set(path4, buf);
|
|
6381
|
+
return { provider: this.provider_tag, native_id: path4, size: buf.length };
|
|
6382
|
+
}
|
|
6383
|
+
async get(path4) {
|
|
6384
|
+
const buf = this.store.get(path4);
|
|
6385
|
+
if (!buf) throw new Error(`Not found: ${path4}`);
|
|
6386
|
+
return buf;
|
|
6387
|
+
}
|
|
6388
|
+
async delete(path4) {
|
|
6389
|
+
this.store.delete(path4);
|
|
6390
|
+
}
|
|
6391
|
+
async exists(path4) {
|
|
6392
|
+
return this.store.has(path4);
|
|
6393
|
+
}
|
|
6394
|
+
async getSignedUrl(path4, _opts) {
|
|
6395
|
+
const buf = this.store.get(path4);
|
|
6396
|
+
if (!buf) throw new Error(`Not found: ${path4}`);
|
|
6397
|
+
return `data:application/octet-stream;base64,${buf.toString("base64")}`;
|
|
6398
|
+
}
|
|
6399
|
+
async probe() {
|
|
6400
|
+
return { ok: true };
|
|
6401
|
+
}
|
|
6402
|
+
/** Test-only escape hatch — exposes the internal store for assertions. */
|
|
6403
|
+
snapshot() {
|
|
6404
|
+
return new Map(this.store);
|
|
6405
|
+
}
|
|
6406
|
+
};
|
|
6407
|
+
async function streamToBuffer2(s) {
|
|
6408
|
+
const chunks = [];
|
|
6409
|
+
for await (const c of s) chunks.push(Buffer.isBuffer(c) ? c : Buffer.from(c));
|
|
6410
|
+
return Buffer.concat(chunks);
|
|
6411
|
+
}
|
|
6412
|
+
|
|
6413
|
+
// src/providers/google-drive.ts
|
|
6414
|
+
import { google as google3 } from "googleapis";
|
|
6415
|
+
var GoogleDriveProvider = class {
|
|
6416
|
+
constructor(opts) {
|
|
6417
|
+
this.provider_tag = "gdrive";
|
|
6418
|
+
let credentials = {};
|
|
6419
|
+
try {
|
|
6420
|
+
credentials = JSON.parse(opts.service_account_json);
|
|
6421
|
+
} catch {
|
|
6422
|
+
}
|
|
6423
|
+
const auth = new google3.auth.GoogleAuth({
|
|
6424
|
+
credentials,
|
|
6425
|
+
scopes: ["https://www.googleapis.com/auth/drive"]
|
|
6426
|
+
});
|
|
6427
|
+
this.drive = google3.drive({ version: "v3", auth });
|
|
6428
|
+
this.driveId = opts.shared_drive_id;
|
|
6429
|
+
this.cache = opts.path_cache;
|
|
6430
|
+
}
|
|
6431
|
+
async probe() {
|
|
6432
|
+
try {
|
|
6433
|
+
await this.drive.drives.get({
|
|
6434
|
+
driveId: this.driveId,
|
|
6435
|
+
supportsAllDrives: true
|
|
6436
|
+
});
|
|
6437
|
+
return { ok: true };
|
|
6438
|
+
} catch (err) {
|
|
6439
|
+
const e = err;
|
|
6440
|
+
if (e?.code === 404) return { ok: false, error: "drive_not_shared", message: "SA cannot see Shared Drive" };
|
|
6441
|
+
if (e?.code === 403) return { ok: false, error: "write_denied", message: "SA lacks permission" };
|
|
6442
|
+
return { ok: false, error: "transient", message: String(err) };
|
|
6443
|
+
}
|
|
6444
|
+
}
|
|
6445
|
+
async put(path4, body, opts) {
|
|
6446
|
+
const segments = path4.split("/").filter(Boolean);
|
|
6447
|
+
const filename = segments.pop();
|
|
6448
|
+
const parentId = await this.resolvePath(segments, { create: true });
|
|
6449
|
+
if (opts?.ifNotExists) {
|
|
6450
|
+
const existing = await this.findChild(filename, parentId);
|
|
6451
|
+
if (existing) throw new Error(`File exists: ${path4}`);
|
|
6452
|
+
}
|
|
6453
|
+
const buf = Buffer.isBuffer(body) ? body : await streamToBuffer3(body);
|
|
6454
|
+
const res = await this.drive.files.create({
|
|
6455
|
+
requestBody: {
|
|
6456
|
+
name: filename,
|
|
6457
|
+
parents: [parentId],
|
|
6458
|
+
mimeType: opts?.contentType ?? "application/octet-stream"
|
|
6459
|
+
},
|
|
6460
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
6461
|
+
media: { body: buf },
|
|
6462
|
+
fields: "id, name"
|
|
6463
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
6464
|
+
});
|
|
6465
|
+
return { provider: this.provider_tag, native_id: res.data.id, size: buf.length };
|
|
6466
|
+
}
|
|
6467
|
+
async get(path4) {
|
|
6468
|
+
const id = await this.lookupFileId(path4);
|
|
6469
|
+
const res = await this.drive.files.get(
|
|
6470
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
6471
|
+
{ fileId: id, alt: "media", supportsAllDrives: true },
|
|
6472
|
+
{ responseType: "arraybuffer" }
|
|
6473
|
+
);
|
|
6474
|
+
return Buffer.from(res.data);
|
|
6475
|
+
}
|
|
6476
|
+
async delete(path4) {
|
|
6477
|
+
try {
|
|
6478
|
+
const id = await this.lookupFileId(path4);
|
|
6479
|
+
await this.drive.files.delete({ fileId: id, supportsAllDrives: true });
|
|
6480
|
+
} catch (err) {
|
|
6481
|
+
const e = err;
|
|
6482
|
+
if (e?.code !== 404) throw err;
|
|
6483
|
+
}
|
|
6484
|
+
}
|
|
6485
|
+
async exists(path4) {
|
|
6486
|
+
try {
|
|
6487
|
+
await this.lookupFileId(path4);
|
|
6488
|
+
return true;
|
|
6489
|
+
} catch {
|
|
6490
|
+
return false;
|
|
6491
|
+
}
|
|
6492
|
+
}
|
|
6493
|
+
async getSignedUrl(path4, _opts) {
|
|
6494
|
+
const id = await this.lookupFileId(path4);
|
|
6495
|
+
return `https://drive.google.com/uc?id=${id}&export=download`;
|
|
6496
|
+
}
|
|
6497
|
+
async resolvePath(segments, opts) {
|
|
6498
|
+
let parentId = this.driveId;
|
|
6499
|
+
const accumulated = [];
|
|
6500
|
+
for (const segment of segments) {
|
|
6501
|
+
accumulated.push(segment);
|
|
6502
|
+
const cacheKey = accumulated.join("/");
|
|
6503
|
+
const cached = await this.cache.lookup(cacheKey);
|
|
6504
|
+
if (cached) {
|
|
6505
|
+
parentId = cached;
|
|
6506
|
+
continue;
|
|
6507
|
+
}
|
|
6508
|
+
let found = await this.findChild(segment, parentId, "application/vnd.google-apps.folder");
|
|
6509
|
+
if (!found && opts.create) {
|
|
6510
|
+
const created = await this.drive.files.create({
|
|
6511
|
+
requestBody: {
|
|
6512
|
+
name: segment,
|
|
6513
|
+
parents: [parentId],
|
|
6514
|
+
mimeType: "application/vnd.google-apps.folder"
|
|
6515
|
+
},
|
|
6516
|
+
fields: "id"
|
|
6517
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
6518
|
+
});
|
|
6519
|
+
found = created.data.id;
|
|
6520
|
+
}
|
|
6521
|
+
if (!found) throw new Error(`Folder not found and create=false: ${cacheKey}`);
|
|
6522
|
+
await this.cache.write(cacheKey, found);
|
|
6523
|
+
parentId = found;
|
|
6524
|
+
}
|
|
6525
|
+
return parentId;
|
|
6526
|
+
}
|
|
6527
|
+
async findChild(name, parentId, mimeType) {
|
|
6528
|
+
const q = [
|
|
6529
|
+
`name='${name.replace(/'/g, "\\'")}'`,
|
|
6530
|
+
`'${parentId}' in parents`,
|
|
6531
|
+
`trashed=false`
|
|
6532
|
+
];
|
|
6533
|
+
if (mimeType) q.push(`mimeType='${mimeType}'`);
|
|
6534
|
+
const res = await this.drive.files.list({
|
|
6535
|
+
q: q.join(" and "),
|
|
6536
|
+
driveId: this.driveId,
|
|
6537
|
+
corpora: "drive",
|
|
6538
|
+
includeItemsFromAllDrives: true,
|
|
6539
|
+
fields: "files(id, name)"
|
|
6540
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
6541
|
+
});
|
|
6542
|
+
return res.data.files?.[0]?.id ?? null;
|
|
6543
|
+
}
|
|
6544
|
+
async lookupFileId(path4) {
|
|
6545
|
+
const segments = path4.split("/").filter(Boolean);
|
|
6546
|
+
const filename = segments.pop();
|
|
6547
|
+
const parentId = await this.resolvePath(segments, { create: false });
|
|
6548
|
+
const id = await this.findChild(filename, parentId);
|
|
6549
|
+
if (!id) throw Object.assign(new Error(`Not found: ${path4}`), { code: 404 });
|
|
6550
|
+
return id;
|
|
6551
|
+
}
|
|
6552
|
+
};
|
|
6553
|
+
async function streamToBuffer3(s) {
|
|
6554
|
+
const chunks = [];
|
|
6555
|
+
for await (const c of s) chunks.push(Buffer.isBuffer(c) ? c : Buffer.from(c));
|
|
6556
|
+
return Buffer.concat(chunks);
|
|
6557
|
+
}
|
|
6558
|
+
|
|
6559
|
+
// src/providers/types.ts
|
|
6560
|
+
var StorageCollisionExhausted = class extends Error {
|
|
6561
|
+
constructor(attempts, lastPath) {
|
|
6562
|
+
super(`Storage collision could not be resolved after ${attempts} attempts at "${lastPath}"`);
|
|
6563
|
+
this.attempts = attempts;
|
|
6564
|
+
this.lastPath = lastPath;
|
|
6565
|
+
this.name = "StorageCollisionExhausted";
|
|
6566
|
+
}
|
|
6567
|
+
};
|
|
6568
|
+
var StorageNotConfigured = class extends Error {
|
|
6569
|
+
constructor() {
|
|
6570
|
+
super("Storage provider is not configured for this scope");
|
|
6571
|
+
this.name = "StorageNotConfigured";
|
|
6572
|
+
}
|
|
6573
|
+
};
|
|
6574
|
+
var StorageUnavailable = class extends Error {
|
|
6575
|
+
constructor(reason, message) {
|
|
6576
|
+
super(message);
|
|
6577
|
+
this.reason = reason;
|
|
6578
|
+
this.name = "StorageUnavailable";
|
|
6579
|
+
}
|
|
6580
|
+
};
|
|
6581
|
+
|
|
6291
6582
|
// src/migrations/add-reference-tracking.ts
|
|
6292
6583
|
async function migrateToV2(executor, dbType, tableName) {
|
|
6293
6584
|
const migration = tableName ? getMigrationForTable(tableName, dbType) : HAZO_FILES_MIGRATION_V2[dbType];
|
|
@@ -6330,6 +6621,7 @@ try {
|
|
|
6330
6621
|
}
|
|
6331
6622
|
export {
|
|
6332
6623
|
ALL_SYSTEM_VARIABLES,
|
|
6624
|
+
AppFileServerProvider,
|
|
6333
6625
|
AuthenticationError,
|
|
6334
6626
|
ConfigurationError,
|
|
6335
6627
|
DEFAULT_DATE_FORMATS,
|
|
@@ -6345,6 +6637,7 @@ export {
|
|
|
6345
6637
|
FileTooLargeError,
|
|
6346
6638
|
GoogleDriveAuth,
|
|
6347
6639
|
GoogleDriveModule,
|
|
6640
|
+
GoogleDriveProvider,
|
|
6348
6641
|
HAZO_FILES_DEFAULT_TABLE_NAME,
|
|
6349
6642
|
HAZO_FILES_JOB_TYPES,
|
|
6350
6643
|
HAZO_FILES_MIGRATION_V2,
|
|
@@ -6357,6 +6650,7 @@ export {
|
|
|
6357
6650
|
HAZO_FILE_QUOTAS_TABLE_SCHEMA,
|
|
6358
6651
|
HazoFilesError,
|
|
6359
6652
|
ImportSizeCapError,
|
|
6653
|
+
InMemoryProvider,
|
|
6360
6654
|
InvalidExtensionError,
|
|
6361
6655
|
InvalidPathError,
|
|
6362
6656
|
LLMExtractionService,
|
|
@@ -6370,6 +6664,9 @@ export {
|
|
|
6370
6664
|
SYSTEM_COUNTER_VARIABLES,
|
|
6371
6665
|
SYSTEM_DATE_VARIABLES,
|
|
6372
6666
|
SYSTEM_FILE_VARIABLES,
|
|
6667
|
+
StorageCollisionExhausted,
|
|
6668
|
+
StorageNotConfigured,
|
|
6669
|
+
StorageUnavailable,
|
|
6373
6670
|
TrackedFileManager,
|
|
6374
6671
|
UploadExtractService,
|
|
6375
6672
|
addExtractionToFileData,
|