@peers-app/peers-sdk 0.30.0 → 0.30.1

package/dist/data/files/file-write-stream.js

@@ -2,7 +2,26 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.FileWriteStream = void 0;
 const keys_1 = require("../../keys");
+const chunk_store_1 = require("./chunk-store");
 const file_types_1 = require("./file.types");
+const file_rows_1 = require("./file-rows");
+/**
+ * `text` as every host's storage keeps it. The id must be derived from the text as stored, or no
+ * other device could rederive it. A lone surrogate (possible in a Windows file name) becomes
+ * U+FFFD. The PWA's sql.js also cuts text at a NUL and drops a leading U+FEFF when reading, so
+ * neither is kept anywhere.
+ */
+function asStored(text) {
+    if (typeof text !== "string")
+        return text;
+    // ignoreBOM leaves U+FEFF for the replace, which drops every leading one (sql.js drops one).
+    const wellFormed = new TextDecoder("utf-8", { ignoreBOM: true }).decode(new TextEncoder().encode(text));
+    return wellFormed.split("\0", 1)[0].replace(/^\uFEFF+/, "");
+}
+/**
+ * Writes one file in pieces: each full chunk is stored as it arrives, and `finalize` saves the
+ * row under the id derived from the content. Open one with {@link FilesTable.createWriteStream}.
+ */
 class FileWriteStream {
     metadata;
     fileTable;
@@ -11,10 +30,20 @@ class FileWriteStream {
     bytesWritten = 0;
     finalized = false;
     aborted = false;
+    /**
+     * @param metadata - The file's name, MIME type and `keep`.
+     * @param fileTable - The Files table the row is saved to.
+     */
     constructor(metadata, fileTable) {
         this.metadata = metadata;
         this.fileTable = fileTable;
     }
+    /**
+     * Append bytes. Full chunks are stored as they complete.
+     *
+     * @param chunk - The next bytes, of any length.
+     * @throws When the file would exceed {@link FILE_MAX_SIZE}, or the stream has ended.
+     */
     async write(chunk) {
         if (this.finalized) {
             throw new Error("Cannot write to finalized stream");
@@ -22,6 +51,9 @@ class FileWriteStream {
         if (this.aborted) {
             throw new Error("Cannot write to aborted stream");
         }
+        if (this.bytesWritten + chunk.length > file_types_1.FILE_MAX_SIZE) {
+            throw new Error(`A file can be at most ${file_types_1.FILE_MAX_SIZE / (1024 * 1024)} MiB`);
+        }
         // Add chunk to buffer
         this.buffer = new Uint8Array(Buffer.concat([this.buffer, chunk]));
         this.bytesWritten += chunk.length;
@@ -29,18 +61,25 @@ class FileWriteStream {
         await this.processCompleteChunks();
     }
     async processCompleteChunks() {
-        const ops = await (0, file_types_1.getFileOps)();
         while (this.buffer.length >= file_types_1.FILE_CHUNK_SIZE) {
-            // Extract one chunk
             const chunkData = this.buffer.subarray(0, file_types_1.FILE_CHUNK_SIZE);
             this.buffer = this.buffer.subarray(file_types_1.FILE_CHUNK_SIZE);
-            // Hash and store the chunk
-            const chunkHash = (0, keys_1.hashBytes)(chunkData);
-            this.chunkHashes.push(chunkHash);
-            const chunkPath = `${file_types_1.CHUNKS_DIR}/${chunkHash}`;
-            await ops.writeFile(chunkPath, chunkData);
+            await this.storeChunk(chunkData);
         }
     }
+    async storeChunk(chunkData) {
+        const chunkHash = (0, keys_1.hashBytes)(chunkData);
+        this.chunkHashes.push(chunkHash);
+        // Stored even when present: a save restores bytes a device has freed, and refreshes the
+        // chunk's age so a sweep can't free it under this save.
+        await (await (0, chunk_store_1.getChunkStore)()).put(chunkHash, chunkData);
+    }
+    /**
+     * Store the last chunk and save the file's row. Saving content that already has a row (the
+     * same bytes, name, MIME type and `keep`) writes no row and returns the stored one.
+     *
+     * @returns The file's row.
+     */
     async finalize() {
         if (this.finalized) {
             throw new Error("Stream already finalized");
@@ -48,45 +87,28 @@ class FileWriteStream {
         if (this.aborted) {
             throw new Error("Cannot finalize aborted stream");
         }
-        // First, process any complete chunks in the buffer
         await this.processCompleteChunks();
-        // Then process any remaining partial data as final chunk
         if (this.buffer.length > 0) {
-            const ops = await (0, file_types_1.getFileOps)();
-            const finalChunkHash = (0, keys_1.hashBytes)(this.buffer);
-            this.chunkHashes.push(finalChunkHash);
-            const chunkPath = `${file_types_1.CHUNKS_DIR}/${finalChunkHash}`;
-            await ops.writeFile(chunkPath, this.buffer);
+            await this.storeChunk(this.buffer);
             this.buffer = new Uint8Array(0);
         }
-        // Create the file record using the same logic as FileTable.saveFile
-        const chunkHashesString = JSON.stringify(this.chunkHashes);
-        const fileHash = (0, keys_1.hashBytes)(Buffer.from(chunkHashesString, "utf8"));
-        let fileRecord;
-        if (this.chunkHashes.length > file_types_1.CHUNK_INDEX_THRESHOLD) {
-            // Large file: use recursive index file
-            const indexFileId = await this.fileTable.createIndexFileRecursively(this.chunkHashes);
-            fileRecord = {
-                ...this.metadata,
-                fileSize: this.bytesWritten,
-                fileHash,
-                indexFileId,
-            };
-        }
-        else {
-            // Small file: store chunk hashes directly
-            fileRecord = {
-                ...this.metadata,
-                fileSize: this.bytesWritten,
-                fileHash,
-                chunkHashes: this.chunkHashes,
-            };
-        }
-        // Save to database
-        await this.fileTable.saveFileRecord(fileRecord);
+        const description = {
+            name: asStored(this.metadata.name),
+            fileSize: this.bytesWritten,
+            fileHash: (0, file_types_1.fileHashOf)(this.chunkHashes),
+            mimeType: asStored(this.metadata.mimeType ?? ""),
+            keep: this.metadata.keep ?? true,
+        };
+        const fileRecord = {
+            fileId: (0, file_rows_1.fileIdFor)(description),
+            ...description,
+            chunkHashes: this.chunkHashes,
+        };
+        const saved = await this.fileTable.saveFileRecord(fileRecord);
         this.finalized = true;
-        return fileRecord;
+        return saved;
     }
+    /** Stop without saving a row. Stored chunks stay until collection frees them. */
     async abort() {
         if (this.finalized) {
             throw new Error("Cannot abort finalized stream");
@@ -94,18 +116,20 @@ class FileWriteStream {
         this.aborted = true;
         this.buffer = Buffer.alloc(0);
         this.chunkHashes = [];
-        // Note: We don't delete chunks since they may be shared with other files
-        // Chunk cleanup should be handled by a separate garbage collection process
     }
+    /** Bytes written so far. */
     getBytesWritten() {
         return this.bytesWritten;
     }
+    /** Chunks stored so far. */
     getChunkCount() {
         return this.chunkHashes.length;
     }
+    /** Whether `finalize` has saved the row. */
     isFinalized() {
         return this.finalized;
     }
+    /** Whether `abort` was called. */
     isAborted() {
         return this.aborted;
     }

package/dist/data/files/file.types.d.ts

@@ -1,5 +1,9 @@
 import { z } from "zod";
 import { type ITableMetaData } from "../orm/types";
+/**
+ * A `Files` row: one file's description and its chunk list. Rows saved since ids became derived
+ * from content have a `000r` id and never change; older rows ("legacy") have random ids.
+ */
 export declare const fileSchema: z.ZodObject<{
     fileId: z.ZodEffects<z.ZodString, string, string>;
     name: z.ZodString;
@@ -7,47 +11,64 @@ export declare const fileSchema: z.ZodObject<{
     fileHash: z.ZodString;
     mimeType: z.ZodOptional<z.ZodString>;
     chunkHashes: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
-    isIndexFile: z.ZodOptional<z.ZodBoolean>;
-    indexFileId: z.ZodOptional<z.ZodEffects<z.ZodString, string, string>>;
+    keep: z.ZodOptional<z.ZodBoolean>;
 }, "strip", z.ZodTypeAny, {
     name: string;
     fileId: string;
     fileSize: number;
     fileHash: string;
+    keep?: boolean | undefined;
     mimeType?: string | undefined;
     chunkHashes?: string[] | undefined;
-    isIndexFile?: boolean | undefined;
-    indexFileId?: string | undefined;
 }, {
     name: string;
     fileId: string;
     fileSize: number;
     fileHash: string;
+    keep?: boolean | undefined;
     mimeType?: string | undefined;
     chunkHashes?: string[] | undefined;
-    isIndexFile?: boolean | undefined;
-    indexFileId?: string | undefined;
 }>;
+/** A `Files` row. */
 export type IFile = z.infer<typeof fileSchema>;
-export type IFileInput = Pick<IFile, "fileId" | "name" | "fileSize" | "mimeType" | "isIndexFile">;
+/**
+ * What a writer chooses about a new file. The id is derived from the content and these, so
+ * saving the same bytes with the same name, MIME type and `keep` returns the same file.
+ */
+export interface IFileInput {
+    /**
+     * Stored as every host can keep it: a lone surrogate becomes U+FFFD, the text ends at a NUL,
+     * and a leading U+FEFF is dropped. The saved row has the name as stored.
+     */
+    name: string;
+    /** Stored like `name`; an omitted MIME type is stored empty. */
+    mimeType?: string;
+    /**
+     * Whether the file is never collected. Defaults to true, so a writer that doesn't declare its
+     * references keeps its files. System Files saves and the package installer pass false.
+     */
+    keep?: boolean;
+}
+/** The `Files` system table: every context has one. */
 export declare const filesMetaData: ITableMetaData;
+/** Size of each chunk a file is split into; the last chunk may be shorter. */
 export declare const FILE_CHUNK_SIZE: number;
-export declare const CHUNKS_DIR = "file_chunks";
-export declare let CHUNK_INDEX_THRESHOLD: number;
-export declare function setChunkIndexThreshold(threshold: number): void;
+/** Most chunks one file may have: a row's chunk list holds every chunk of any file. */
+export declare const FILE_MAX_CHUNKS = 1024;
+/** Largest file Peers stores (1 GiB). It can rise in a later release, never fall. */
+export declare const FILE_MAX_SIZE: number;
+/** Most chunks a read fetches from other devices at once, so a large file isn't one stream. */
+export declare const FILE_FETCH_WINDOW = 16;
+/**
+ * A file's `fileHash`: the SHA-256 of its chunk hashes as JSON, in order.
+ *
+ * @param chunkHashes - The file's chunk hashes.
+ */
+export declare function fileHashOf(chunkHashes: readonly string[]): string;
 /**
- * Compute the file-system hash for content without saving it.
- * Mirrors the chunking + hashing logic of FileWriteStream so the result
- * matches `IFile.fileHash` for the same bytes.
+ * Compute the file-system hash for content without saving it. Mirrors the chunking of
+ * `FileWriteStream`, so the result matches `IFile.fileHash` for the same bytes.
+ *
+ * @param content - The bytes, or a string hashed as UTF-8.
  */
 export declare function computeFileHash(content: string | Uint8Array): string;
-export interface FileOps {
-    downloadFileChunk(chunkHash: string): Promise<Uint8Array | null>;
-    fileExists(path: string): Promise<boolean>;
-    readFile(path: string): Promise<Uint8Array>;
-    writeFile(path: string, data: Uint8Array): Promise<void>;
-    deletePath(path: string): Promise<void>;
-}
-export declare function setFileOps(ops: FileOps): void;
-export declare function getFileOps(): Promise<FileOps>;
-export declare function resetFileOps(): void;

package/dist/data/files/file.types.js

@@ -1,80 +1,62 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.CHUNK_INDEX_THRESHOLD = exports.CHUNKS_DIR = exports.FILE_CHUNK_SIZE = exports.filesMetaData = exports.fileSchema = void 0;
-exports.setChunkIndexThreshold = setChunkIndexThreshold;
+exports.FILE_FETCH_WINDOW = exports.FILE_MAX_SIZE = exports.FILE_MAX_CHUNKS = exports.FILE_CHUNK_SIZE = exports.filesMetaData = exports.fileSchema = void 0;
+exports.fileHashOf = fileHashOf;
 exports.computeFileHash = computeFileHash;
-exports.setFileOps = setFileOps;
-exports.getFileOps = getFileOps;
-exports.resetFileOps = resetFileOps;
 const zod_1 = require("zod");
 const keys_1 = require("../../keys");
 const zod_types_1 = require("../../types/zod-types");
 const types_1 = require("../orm/types");
+/**
+ * A `Files` row: one file's description and its chunk list. Rows saved since ids became derived
+ * from content have a `000r` id and never change; older rows ("legacy") have random ids.
+ */
 exports.fileSchema = zod_1.z.object({
     fileId: zod_types_1.zodPeerId,
     name: zod_1.z.string().describe("The name of the file"),
     fileSize: zod_1.z.number().describe("The size of the file in bytes"),
     fileHash: zod_1.z.string().describe("Hash of the chunk hashes array for integrity verification"),
-    mimeType: zod_1.z.string().optional().describe("The MIME type of the file"),
-    chunkHashes: zod_1.z
-        .array(zod_1.z.string())
-        .optional()
-        .describe("SHA-256 hashes of each chunk in order (for small files)"),
-    isIndexFile: zod_1.z
+    mimeType: zod_1.z.string().optional().describe("The MIME type of the file; empty when unknown"),
+    chunkHashes: zod_1.z.array(zod_1.z.string()).optional().describe("SHA-256 hashes of each chunk in order"),
+    keep: zod_1.z
         .boolean()
         .optional()
-        .describe("True if this is an index file for another large file"),
-    indexFileId: zod_types_1.zodPeerId.optional().describe("FileId of index file (for large files)"),
+        .describe("True for a file that is never collected, even with nothing referencing it"),
 });
+/** The `Files` system table: every context has one. */
 exports.filesMetaData = {
     name: "Files",
     description: "Files stored in the chunked file system for peer sharing",
     primaryKeyName: "fileId",
     fields: (0, types_1.schemaToFields)(exports.fileSchema),
 };
-// File storage configuration
-exports.FILE_CHUNK_SIZE = 1024 * 1024; // 1MB chunks
-exports.CHUNKS_DIR = "file_chunks";
-exports.CHUNK_INDEX_THRESHOLD = 1000; // Use chunk index file for files with >1000 chunks (~1GB)
-// For testing - allow modifying the threshold
-function setChunkIndexThreshold(threshold) {
-    exports.CHUNK_INDEX_THRESHOLD = threshold;
+/** Size of each chunk a file is split into; the last chunk may be shorter. */
+exports.FILE_CHUNK_SIZE = 1024 * 1024;
+/** Most chunks one file may have: a row's chunk list holds every chunk of any file. */
+exports.FILE_MAX_CHUNKS = 1024;
+/** Largest file Peers stores (1 GiB). It can rise in a later release, never fall. */
+exports.FILE_MAX_SIZE = exports.FILE_MAX_CHUNKS * exports.FILE_CHUNK_SIZE;
+/** Most chunks a read fetches from other devices at once, so a large file isn't one stream. */
+exports.FILE_FETCH_WINDOW = 16;
+/**
+ * A file's `fileHash`: the SHA-256 of its chunk hashes as JSON, in order.
+ *
+ * @param chunkHashes - The file's chunk hashes.
+ */
+function fileHashOf(chunkHashes) {
+    return (0, keys_1.hashBytes)(new TextEncoder().encode(JSON.stringify(chunkHashes)));
 }
 /**
- * Compute the file-system hash for content without saving it.
- * Mirrors the chunking + hashing logic of FileWriteStream so the result
- * matches `IFile.fileHash` for the same bytes.
+ * Compute the file-system hash for content without saving it. Mirrors the chunking of
+ * `FileWriteStream`, so the result matches `IFile.fileHash` for the same bytes.
+ *
+ * @param content - The bytes, or a string hashed as UTF-8.
  */
 function computeFileHash(content) {
     const data = typeof content === "string" ? new Uint8Array(Buffer.from(content, "utf8")) : content;
     const chunkHashes = [];
-    let offset = 0;
-    while (offset < data.length) {
-        const end = Math.min(offset + exports.FILE_CHUNK_SIZE, data.length);
-        chunkHashes.push((0, keys_1.hashBytes)(data.subarray(offset, end)));
-        offset = end;
+    for (let offset = 0; offset < data.length; offset += exports.FILE_CHUNK_SIZE) {
+        chunkHashes.push((0, keys_1.hashBytes)(data.subarray(offset, offset + exports.FILE_CHUNK_SIZE)));
     }
-    return (0, keys_1.hashBytes)(new Uint8Array(Buffer.from(JSON.stringify(chunkHashes), "utf8")));
-}
-let fileOps = null;
-let fileOpsReady = null;
-function setFileOps(ops) {
-    fileOps = ops;
-    if (fileOpsReady) {
-        fileOpsReady(ops);
-        fileOpsReady = null; // Reset the promise if it was waiting
-    }
-}
-function getFileOps() {
-    if (fileOps) {
-        return Promise.resolve(fileOps);
-    }
-    return new Promise((resolve) => {
-        fileOpsReady = resolve;
-    });
-}
-// For testing - reset the global state
-function resetFileOps() {
-    fileOps = null;
-    fileOpsReady = null;
+    return fileHashOf(chunkHashes);
 }

package/dist/data/files/files.d.ts

@@ -5,8 +5,10 @@ import { FileReadStream } from "./file-read-stream";
 import { FileWriteStream } from "./file-write-stream";
 /**
  * The chunked, content-hashed file store of one data context. Rows describe files; the bytes
- * live in chunks on disk, shared by hash, and a device fetches a missing chunk from a connected
- * device when it reads. Write through {@link FilesTable.saveFile}, never the table methods.
+ * live in chunks in the account's chunk store, shared by hash, and a device fetches a missing
+ * chunk from a connected device when it reads. A file's id is derived from its content, and its row never changes.
+ * Write through {@link FilesTable.saveFile}, never the table methods. The rules are checked here
+ * in every context, and by every device on the rows it receives in a group (`file-rows.ts`).
  */
 export declare class FilesTable extends Table<IFile> {
     /** @deprecated Direct inserts forbidden; use saveFile() or saveFileRecord() */
@@ -20,14 +22,18 @@ export declare class FilesTable extends Table<IFile> {
     /**
      * Start writing a file in chunks; `finalize()` saves its row.
      *
-     * @param metadata - The new file's id, name, size and MIME type.
+     * @param metadata - The new file's name, MIME type and `keep`.
      */
     createWriteStream(metadata: IFileInput): Promise<FileWriteStream>;
     /**
-     * Note: Use `saveFile` to write a file to disk. This method is for managing
-     * the file metadata in the database which is done automatically by `saveFile`.
-     * @param fileRecord The file record to save to the database
-     * @returns The inserted file record
+     * Store a file's row as it is, after the checks every device makes: a new row must have the
+     * id derived from its content (or be a legacy row), and a stored row never changes. Storing
+     * the same file again writes nothing. Writers use {@link saveFile} or a write stream; this is
+     * for copying a row from another context with its id.
+     *
+     * @param fileRecord - The row.
+     * @returns The row as stored.
+     * @throws When the row is refused, or a different row has its id.
      */
     saveFileRecord(fileRecord: IFile): Promise<IFile>;
     /**
@@ -40,11 +46,12 @@ export declare class FilesTable extends Table<IFile> {
      */
     openReadStream(fileId: string, preloadChunksCount?: number, skipVerification?: boolean): Promise<FileReadStream | null>;
     /**
-     * Save a whole file: chunk it, store the chunks and write its row.
+     * Save a whole file: chunk it, store the chunks and write its row. The same bytes, name, MIME
+     * type and `keep` give the same file, whose chunks are stored again.
      *
-     * @param metaData - The file's id, name and MIME type. `fileSize` is taken from `data`.
+     * @param metaData - The file's name, MIME type and `keep` (true unless given).
      * @param data - The bytes, or a string saved as UTF-8.
-     * @returns The saved row.
+     * @returns The file's row.
      */
     saveFile(metaData: IFileInput, data: Uint8Array | string): Promise<IFile>;
     /**
@@ -58,25 +65,21 @@ export declare class FilesTable extends Table<IFile> {
         skipVerification?: boolean;
     }): Promise<Uint8Array | null>;
     /**
-     * Delete a file's row, and its index file's. Chunks stay on disk: other files may share them.
-     *
-     * @param fileId - File to delete. A missing file is not an error.
-     */
-    deleteFile(fileId: string): Promise<void>;
-    /**
-     * Save a large file's chunk hashes as an index file, itself chunked when large.
+     * Make sure this device has every chunk of a file, fetching the missing ones from connected
+     * devices {@link FILE_FETCH_WINDOW} at a time, without reading the file into memory.
      *
-     * @param chunkHashes - The large file's chunk hashes, in order.
-     * @returns The index file's id.
+     * @param fileId - File to fetch.
+     * @returns Whether every chunk is now on this device; false for a file with no row.
      */
-    createIndexFileRecursively(chunkHashes: string[]): Promise<string>;
+    fetchFile(fileId: string): Promise<boolean>;
     /**
-     * Read a large file's chunk hashes back from its index file.
+     * Delete a legacy bundle row, which the runtime owns. Every other file is permanent: it's
+     * collected once nothing references it. Chunks stay stored, since other files may share them.
      *
-     * @param indexFileId - The index file's id.
-     * @returns The chunk hashes, in order.
+     * @param fileId - File to delete. A missing file is not an error.
+     * @throws When the file isn't a legacy bundle row.
      */
-    loadChunkHashesRecursively(indexFileId: string): Promise<string[]>;
+    deleteFile(fileId: string): Promise<void>;
 }
 /**
  * The Files table of a data context.