@lde/distribution-downloader 0.7.0 → 0.9.0

package/dist/download.d.ts

@@ -32,6 +32,18 @@ export declare class LastModifiedDownloader implements Downloader {
     private readonly path;
     constructor(path?: string);
     download(distribution: Distribution, target?: string, options?: DownloadOptions): Promise<DownloadResult>;
+    /**
+     * Copy a local file into the base directory, because that is the only
+     * directory a containerized store has mounted. An unchanged copy is kept: it
+     * is what keeps a store’s index of it valid.
+     *
+     * The copy is unchanged when the source still has the size and modification
+     * time it had when copied, as recorded beside the copy. Comparing with the
+     * copy’s own time would miss a replacement restored with an older time. The
+     * copy keeps the time of copying, so a store indexing by modification time
+     * sees it as new.
+     */
+    private copyLocalFile;
     private localFileIsUpToDate;
 }
 //# sourceMappingURL=download.d.ts.map

package/dist/download.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../src/download.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAO5C,MAAM,WAAW,MAAM;IACrB,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC;IAC7C,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC;IAC7C,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC;IAC5C,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC;IAC5C,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC;IAC7C,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC;CAC9C;AAaD,MAAM,WAAW,eAAe;IAC9B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;;OAKG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;OAEG;IACH,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,UAAU;IACzB,QAAQ,CACN,YAAY,EAAE,YAAY,EAC1B,MAAM,CAAC,EAAE,MAAM,EACf,OAAO,CAAC,EAAE,eAAe,GACxB,OAAO,CAAC,cAAc,CAAC,CAAC;CAC5B;AAED,qBAAa,sBAAuB,YAAW,UAAU;IAC3C,OAAO,CAAC,QAAQ,CAAC,IAAI;gBAAJ,IAAI,SAAY;IAEhC,QAAQ,CACnB,YAAY,EAAE,YAAY,EAC1B,MAAM,SAAyD,EAC/D,OAAO,CAAC,EAAE,eAAe,GACxB,OAAO,CAAC,cAAc,CAAC;YA0EZ,mBAAmB;CAyBlC"}
+{"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../src/download.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAQ5C,MAAM,WAAW,MAAM;IACrB,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC;IAC7C,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC;IAC7C,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC;IAC5C,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC;IAC5C,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC;IAC7C,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC;CAC9C;AAaD,MAAM,WAAW,eAAe;IAC9B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;;OAKG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;OAEG;IACH,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,UAAU;IACzB,QAAQ,CACN,YAAY,EAAE,YAAY,EAC1B,MAAM,CAAC,EAAE,MAAM,EACf,OAAO,CAAC,EAAE,eAAe,GACxB,OAAO,CAAC,cAAc,CAAC,CAAC;CAC5B;AAED,qBAAa,sBAAuB,YAAW,UAAU;IAC3C,OAAO,CAAC,QAAQ,CAAC,IAAI;gBAAJ,IAAI,SAAY;IAEhC,QAAQ,CACnB,YAAY,EAAE,YAAY,EAC1B,MAAM,SAKL,EACD,OAAO,CAAC,EAAE,eAAe,GACxB,OAAO,CAAC,cAAc,CAAC;IAmF1B;;;;;;;;;;OAUG;YACW,aAAa;YAwCb,mBAAmB;CAyBlC"}

package/dist/download.js

@@ -1,9 +1,10 @@
 import filenamifyUrl from 'filenamify-url';
-import { dirname, join, resolve, sep } from 'node:path';
+import { createHash } from 'node:crypto';
+import { basename, dirname, join, resolve, sep } from 'node:path';
 import { Transform } from 'node:stream';
 import { pipeline } from 'node:stream/promises';
-import { createWriteStream } from 'node:fs';
-import { access, mkdir, rm, stat } from 'node:fs/promises';
+import { createReadStream, createWriteStream } from 'node:fs';
+import { access, mkdir, readFile, rm, stat, writeFile } from 'node:fs/promises';
 // eslint-disable-next-line @typescript-eslint/no-empty-function
 const noop = () => { };
 const noopLogger = {
@@ -19,7 +20,9 @@ export class LastModifiedDownloader {
     constructor(path = 'imports') {
         this.path = path;
     }
-    async download(distribution, target = join(this.path, filenamifyUrl(distribution.accessUrl)), options) {
+    async download(distribution, target = join(this.path, distribution.localPath === undefined
+        ? filenamifyUrl(distribution.accessUrl)
+        : localCopyName(distribution.localPath)), options) {
         const logger = options?.logger ?? noopLogger;
         const downloadUrl = distribution.accessUrl;
         const filePath = resolve(target);
@@ -27,6 +30,10 @@ export class LastModifiedDownloader {
         if (!filePath.startsWith(baseDir + sep)) {
             throw new Error(`Download target escapes the base directory: ${filePath}`);
         }
+        if (distribution.localPath !== undefined) {
+            await this.copyLocalFile(distribution.localPath, filePath, options?.signal);
+            return { path: filePath, headers: new Headers() };
+        }
         if (await this.localFileIsUpToDate(filePath, distribution)) {
             logger.debug(`File ${filePath} is up to date, skipping download.`);
             return { path: filePath, headers: new Headers() };
@@ -71,6 +78,50 @@ export class LastModifiedDownloader {
             clearTimeout(idleTimer);
         }
     }
+    /**
+     * Copy a local file into the base directory, because that is the only
+     * directory a containerized store has mounted. An unchanged copy is kept: it
+     * is what keeps a store’s index of it valid.
+     *
+     * The copy is unchanged when the source still has the size and modification
+     * time it had when copied, as recorded beside the copy. Comparing with the
+     * copy’s own time would miss a replacement restored with an older time. The
+     * copy keeps the time of copying, so a store indexing by modification time
+     * sees it as new.
+     */
+    async copyLocalFile(source, filePath, signal) {
+        const sourceStats = await stat(source);
+        if (sourceStats.size <= 1) {
+            throw new Error(`Local file ${source} is empty`);
+        }
+        const recordPath = `${filePath}.source.json`;
+        const record = JSON.stringify({
+            size: sourceStats.size,
+            mtimeMs: sourceStats.mtimeMs,
+        });
+        const [copied, previousRecord] = await Promise.all([
+            access(filePath).then(() => true, () => false),
+            readFile(recordPath, 'utf8').catch(() => undefined),
+        ]);
+        if (copied && previousRecord === record) {
+            return;
+        }
+        await mkdir(dirname(filePath), { recursive: true });
+        try {
+            // Streamed rather than copyFile, which would carry the source’s mode over:
+            // a read-only source would block the next copy, and a private one leave a
+            // copy a store running as another user cannot read.
+            await pipeline(createReadStream(source), createWriteStream(filePath), {
+                signal,
+            });
+        }
+        catch (error) {
+            await rm(filePath, { force: true });
+            await rm(recordPath, { force: true });
+            throw new Error(`Failed to copy ${source} to ${filePath}: ${error}`);
+        }
+        await writeFile(recordPath, record);
+    }
     async localFileIsUpToDate(filePath, distribution) {
         if (undefined === distribution.lastModified) {
             return false;
@@ -90,3 +141,15 @@ export class LastModifiedDownloader {
         return stats.mtime >= distribution.lastModified;
     }
 }
+/**
+ * A copy’s name, unique per source path: a filename derived from the path would
+ * be lossy, so `/rdf/a/b.ttl` and `/rdf/a!b.ttl` would share one copy and one
+ * could be indexed as the other. The extensions are kept for the store, which
+ * reads the format and compression from them.
+ */
+function localCopyName(path) {
+    const hash = createHash('sha256').update(path).digest('hex').slice(0, 16);
+    // Always matches, if only the empty string at the end.
+    const extensions = /(\.[a-z0-9]+){0,2}$/i.exec(basename(path))[0];
+    return `local-${hash}${extensions}`;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/distribution-downloader",
-  "version": "0.7.0",
+  "version": "0.9.0",
   "repository": {
     "url": "git+https://github.com/ldelements/lde.git",
     "directory": "packages/distribution-downloader"
@@ -24,7 +24,7 @@
     "!**/*.tsbuildinfo"
   ],
   "dependencies": {
-    "@lde/dataset": "^0.8.0",
+    "@lde/dataset": "^0.10.0",
     "filenamify-url": "4.0.0",
     "tslib": "^2.3.0"
   }