@chunkd/source 11.0.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 Blayne Chard
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,49 @@
+# @chunkd/source
+
+Base interface for all sources used by `@chunkd`
+
+````typescript
+export interface Source {
+  /**
+   *  human friendly name of the source type
+   * @example "aws:s3", "file", "memory"
+   */
+  type: string;
+
+  /**
+   * URL to the source
+   *
+   * @example "s3://linz-imagery/catalog.json" or "file:///home/blacha/18_126359_137603.webp"
+   */
+  url: URL;
+
+  /** Metadata about the object the source represents */
+  metadata?: SourceMetadata;
+
+  /** Read the metadata before a fetch request */
+  head(): Promise<SourceMetadata>;
+
+  /**
+   * Directly read bytes from the source
+   *
+   * @example
+   * ```typescript
+   *  source.fetch(0, 1024) // load the first 1024 bytes
+   *  source.fetch(1024, 20) // read 20 bytes at offset 1024
+   *  source.fetch(-1024) // load the last 1024 bytes
+   *```
+   * @param offset Byte to start reading form
+   * @param length optional number of bytes to read
+   */
+  fetch(offset: number, length?: number): Promise<ArrayBuffer>;
+}
+````
+
+## Examples
+
+Example sources
+
+- [Memory](https://www.npmjs.com/package/@chunkd/source-memory)
+- [File](https://www.npmjs.com/package/@chunkd/source-file)
+- [AWS](https://www.npmjs.com/package/@chunkd/source-aws)
+- [Http](https://www.npmjs.com/package/@chunkd/source-http)

package/build/src/__test__/range.test.d.ts

@@ -0,0 +1 @@
+export {};

package/build/src/__test__/range.test.js

@@ -0,0 +1,18 @@
+import assert from 'node:assert';
+import { describe, it } from 'node:test';
+import { ContentRange } from '../range.js';
+describe('ContentRange', () => {
+    it('should create a range request', () => {
+        assert.equal(ContentRange.toRange(0, 1024), 'bytes=0-1023');
+    });
+    it('should parse size from range response', () => {
+        assert.equal(ContentRange.parseSize('bytes 0-1024/46661'), 46661);
+        assert.equal(ContentRange.parseSize('bytes */30'), 30);
+    });
+    it('should throw on badly formated ranges', () => {
+        assert.throws(() => ContentRange.parseSize(''));
+        assert.throws(() => ContentRange.parseSize('test 1234'));
+        assert.throws(() => ContentRange.parseSize('bytes 0-1024/*'));
+    });
+});
+//# sourceMappingURL=range.test.js.map

package/build/src/__test__/range.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"range.test.js","sourceRoot":"","sources":["../../../src/__test__/range.test.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,WAAW,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;IAC5B,EAAE,CAAC,+BAA+B,EAAE,GAAG,EAAE;QACvC,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,cAAc,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;QAC/C,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,SAAS,CAAC,oBAAoB,CAAC,EAAE,KAAK,CAAC,CAAC;QAClE,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,SAAS,CAAC,YAAY,CAAC,EAAE,EAAE,CAAC,CAAC;IACzD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;QAC/C,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,YAAY,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC;QAChD,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,YAAY,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC,CAAC;QACzD,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,YAAY,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAC,CAAC;IAChE,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/build/src/error.d.ts

@@ -0,0 +1,12 @@
+import { Source } from './source.js';
+export declare class SourceError extends Error {
+    /** Status code for the error eg 403 - Forbidden vs 404 Not found */
+    readonly code: number;
+    /** URL that the status code came from */
+    readonly url: URL;
+    /** File system that triggered the error */
+    readonly source: Source;
+    constructor(msg: string, code: number, source: Source, cause?: unknown);
+    /** Check that a unknown is a FsError */
+    static is(e: unknown): e is SourceError;
+}

package/build/src/error.js

@@ -0,0 +1,31 @@
+export class SourceError extends Error {
+    /** Status code for the error eg 403 - Forbidden vs 404 Not found */
+    code;
+    /** URL that the status code came from */
+    url;
+    /** File system that triggered the error */
+    source;
+    constructor(msg, code, source, cause) {
+        super(msg, { cause });
+        this.code = code;
+        this.url = source.url;
+        this.source = source;
+        /** Helper to determine if this class is a SourceError, useful for when `instanceof` fails */
+        Object.defineProperty(this, '_sourceError', {
+            enumerable: false,
+            value: 'SourceError',
+        });
+    }
+    /** Check that a unknown is a FsError */
+    static is(e) {
+        if (e instanceof SourceError)
+            return true;
+        // Sometimes instanceof fails, so fall back to checking for a `_sourceError` key
+        if (typeof e !== 'object')
+            return false;
+        if (e == null)
+            return false;
+        return '_sourceError' in e && e['_sourceError'] === 'SourceError';
+    }
+}
+//# sourceMappingURL=error.js.map

package/build/src/error.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"error.js","sourceRoot":"","sources":["../../src/error.ts"],"names":[],"mappings":"AAEA,MAAM,OAAO,WAAY,SAAQ,KAAK;IACpC,oEAAoE;IAC3D,IAAI,CAAS;IACtB,yCAAyC;IAChC,GAAG,CAAM;IAClB,2CAA2C;IAClC,MAAM,CAAS;IAExB,YAAY,GAAW,EAAE,IAAY,EAAE,MAAc,EAAE,KAAe;QACpE,KAAK,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QACtB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC;QACtB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QAErB,6FAA6F;QAC7F,MAAM,CAAC,cAAc,CAAC,IAAI,EAAE,cAAc,EAAE;YAC1C,UAAU,EAAE,KAAK;YACjB,KAAK,EAAE,aAAa;SACrB,CAAC,CAAC;IACL,CAAC;IAED,wCAAwC;IACxC,MAAM,CAAC,EAAE,CAAC,CAAU;QAClB,IAAI,CAAC,YAAY,WAAW;YAAE,OAAO,IAAI,CAAC;QAC1C,gFAAgF;QAChF,IAAI,OAAO,CAAC,KAAK,QAAQ;YAAE,OAAO,KAAK,CAAC;QACxC,IAAI,CAAC,IAAI,IAAI;YAAE,OAAO,KAAK,CAAC;QAC5B,OAAO,cAAc,IAAI,CAAC,IAAI,CAAC,CAAC,cAAc,CAAC,KAAK,aAAa,CAAC;IACpE,CAAC;CACF"}

package/build/src/index.d.ts

@@ -0,0 +1,6 @@
+export { SourceError } from './error.js';
+export { SourceCallback, SourceMiddleware, SourceRequest } from './middleware.js';
+export { ContentRange } from './range.js';
+export { Source, SourceMetadata } from './source.js';
+export { SourceView } from './view.js';
+export declare function tryParseUrl(s: string): URL | null;

package/build/src/index.js

@@ -0,0 +1,12 @@
+export { SourceError } from './error.js';
+export { ContentRange } from './range.js';
+export { SourceView } from './view.js';
+export function tryParseUrl(s) {
+    try {
+        return new URL(s);
+    }
+    catch (e) {
+        return null;
+    }
+}
+//# sourceMappingURL=index.js.map

package/build/src/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAEzC,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAE1C,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAEvC,MAAM,UAAU,WAAW,CAAC,CAAS;IACnC,IAAI;QACF,OAAO,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC;KACnB;IAAC,OAAO,CAAC,EAAE;QACV,OAAO,IAAI,CAAC;KACb;AACH,CAAC"}

package/build/src/middleware.d.ts

@@ -0,0 +1,11 @@
+import { Source } from './source.js';
+export interface SourceRequest {
+    source: Source;
+    offset: number;
+    length: number | undefined;
+}
+export type SourceCallback = (req: SourceRequest) => Promise<ArrayBuffer>;
+export interface SourceMiddleware {
+    name: string;
+    fetch(req: SourceRequest, next: SourceCallback): Promise<ArrayBuffer>;
+}

package/build/src/middleware.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=middleware.js.map

package/build/src/middleware.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"middleware.js","sourceRoot":"","sources":["../../src/middleware.ts"],"names":[],"mappings":""}

package/build/src/range.d.ts

@@ -0,0 +1,10 @@
+export declare const ContentRange: {
+    /** Convert a offset/length to a range request */
+    toRange(offset: number, length?: number): string;
+    /**
+     * Parse a common Content-Range header to extract the size of the source
+     * @example Content-Range: bytes 200-1000/67589
+     * @throws if range is not a Content-Range with size
+     */
+    parseSize(range: string): number;
+};

package/build/src/range.js

@@ -0,0 +1,28 @@
+export const ContentRange = {
+    /** Convert a offset/length to a range request */
+    toRange(offset, length) {
+        if (length == null)
+            return `bytes=${offset}`;
+        if (offset < 0)
+            throw new Error('Cannot read from remote source with negative offset and length');
+        return `bytes=${offset}-${offset + length - 1}`;
+    },
+    /**
+     * Parse a common Content-Range header to extract the size of the source
+     * @example Content-Range: bytes 200-1000/67589
+     * @throws if range is not a Content-Range with size
+     */
+    parseSize(range) {
+        const [unit, chunks] = range.split(' ');
+        if (unit !== 'bytes')
+            throw new Error('Failed to parse content-range: ' + range);
+        if (chunks == null)
+            throw new Error('Failed to parse content-range: ' + range);
+        const [, size] = chunks.split('/');
+        const result = Number(size);
+        if (isNaN(result))
+            throw new Error('Failed to parse content-range: ' + range);
+        return result;
+    },
+};
+//# sourceMappingURL=range.js.map

package/build/src/range.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"range.js","sourceRoot":"","sources":["../../src/range.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,YAAY,GAAG;IAC1B,iDAAiD;IACjD,OAAO,CAAC,MAAc,EAAE,MAAe;QACrC,IAAI,MAAM,IAAI,IAAI;YAAE,OAAO,SAAS,MAAM,EAAE,CAAC;QAC7C,IAAI,MAAM,GAAG,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,gEAAgE,CAAC,CAAC;QAClG,OAAO,SAAS,MAAM,IAAI,MAAM,GAAG,MAAM,GAAG,CAAC,EAAE,CAAC;IAClD,CAAC;IAED;;;;OAIG;IACH,SAAS,CAAC,KAAa;QACrB,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACxC,IAAI,IAAI,KAAK,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,iCAAiC,GAAG,KAAK,CAAC,CAAC;QACjF,IAAI,MAAM,IAAI,IAAI;YAAE,MAAM,IAAI,KAAK,CAAC,iCAAiC,GAAG,KAAK,CAAC,CAAC;QAC/E,MAAM,CAAC,EAAE,IAAI,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACnC,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QAC5B,IAAI,KAAK,CAAC,MAAM,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,iCAAiC,GAAG,KAAK,CAAC,CAAC;QAC9E,OAAO,MAAM,CAAC;IAChB,CAAC;CACF,CAAC"}

package/build/src/source.d.ts

@@ -0,0 +1,43 @@
+export interface Source {
+    /**
+     *  human friendly name of the source type
+     * @example "aws:s3", "file", "memory"
+     */
+    type: string;
+    /**
+     * URL to the source
+     *
+     * @example "s3://linz-imagery/catalog.json" or "file:///home/blacha/18_126359_137603.webp"
+     */
+    url: URL;
+    /** Metadata about the object the source represents */
+    metadata?: SourceMetadata;
+    /** Read the metadata before a fetch request */
+    head(): Promise<SourceMetadata>;
+    /** close the source, sources like files sometimes have open file handles that need to be closed */
+    close?(): Promise<void>;
+    /**
+     * Directly read bytes from the source
+     *
+     * @example
+     * ```typescript
+     *  source.fetch(0, 1024) // load the first 1024 bytes
+     *  source.fetch(1024, 20) // read 20 bytes at offset 1024
+     *  source.fetch(-1024) // load the last 1024 bytes
+     *```
+     * @param offset Byte to start reading form
+     * @param length optional number of bytes to read
+     */
+    fetch(offset: number, length?: number): Promise<ArrayBuffer>;
+}
+/** Metadata returned from some sources like HTTP or AWS */
+export interface SourceMetadata {
+    /** number of bytes of the file */
+    size?: number;
+    /** Entity tag */
+    eTag?: string;
+    contentType?: string;
+    contentDisposition?: string;
+    lastModified?: string;
+    metadata?: Record<string, unknown>;
+}

package/build/src/source.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=source.js.map

package/build/src/source.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"source.js","sourceRoot":"","sources":["../../src/source.ts"],"names":[],"mappings":""}

package/build/src/view.d.ts

@@ -0,0 +1,21 @@
+import { SourceCallback, SourceMiddleware } from './middleware.js';
+import { Source, SourceMetadata } from './source.js';
+/**
+ * Wrap a source with middleware to modify requests to the sources
+ *
+ * @see @chunkd/middleware
+ */
+export declare class SourceView implements Source {
+    source: Source;
+    middleware: SourceMiddleware[];
+    static is(s: Source): s is SourceView;
+    constructor(source: Source, middleware?: SourceMiddleware[]);
+    get type(): string;
+    get url(): URL;
+    get metadata(): SourceMetadata | undefined;
+    head(): Promise<SourceMetadata>;
+    close(): Promise<void>;
+    fetch(offset: number, length?: number): Promise<ArrayBuffer>;
+    /** Run a request using all the middleware */
+    run(handler: SourceCallback, offset: number, length?: number): Promise<ArrayBuffer>;
+}

package/build/src/view.js

@@ -0,0 +1,53 @@
+/**
+ * Wrap a source with middleware to modify requests to the sources
+ *
+ * @see @chunkd/middleware
+ */
+export class SourceView {
+    source;
+    middleware;
+    static is(s) {
+        if ('middleware' in s)
+            return true;
+        return false;
+    }
+    constructor(source, middleware = []) {
+        this.source = source;
+        this.middleware = middleware;
+    }
+    get type() {
+        return this.source.type;
+    }
+    get url() {
+        return this.source.url;
+    }
+    get metadata() {
+        return this.source.metadata;
+    }
+    head() {
+        return this.source.head();
+    }
+    async close() {
+        return this.source.close?.();
+    }
+    async fetch(offset, length) {
+        const middleware = this.middleware;
+        if (middleware == null || middleware.length === 0)
+            return this.source.fetch(offset, length);
+        const handler = (req) => this.source.fetch(req.offset, req.length);
+        return this.run(handler, offset, length);
+    }
+    /** Run a request using all the middleware */
+    async run(handler, offset, length) {
+        const middleware = this.middleware;
+        if (middleware == null)
+            return handler({ source: this, offset, length });
+        function runMiddleware(middleware, next) {
+            return (req) => middleware.fetch(req, next);
+        }
+        for (let i = middleware.length - 1; i >= 0; i--)
+            handler = runMiddleware(middleware[i], handler);
+        return handler({ source: this, offset, length });
+    }
+}
+//# sourceMappingURL=view.js.map

package/build/src/view.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"view.js","sourceRoot":"","sources":["../../src/view.ts"],"names":[],"mappings":"AAGA;;;;GAIG;AACH,MAAM,OAAO,UAAU;IACrB,MAAM,CAAS;IACf,UAAU,CAAqB;IAE/B,MAAM,CAAC,EAAE,CAAC,CAAS;QACjB,IAAI,YAAY,IAAI,CAAC;YAAE,OAAO,IAAI,CAAC;QACnC,OAAO,KAAK,CAAC;IACf,CAAC;IAED,YAAY,MAAc,EAAE,aAAiC,EAAE;QAC7D,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;IAC/B,CAAC;IAED,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;IAC1B,CAAC;IAED,IAAI,GAAG;QACL,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC;IACzB,CAAC;IAED,IAAI,QAAQ;QACV,OAAO,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;IAC9B,CAAC;IAED,IAAI;QACF,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;IAC5B,CAAC;IAED,KAAK,CAAC,KAAK;QACT,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,CAAC;IAC/B,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,MAAc,EAAE,MAAe;QACzC,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QACnC,IAAI,UAAU,IAAI,IAAI,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC5F,MAAM,OAAO,GAAmB,CAAC,GAAkB,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;QAClG,OAAO,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IAC3C,CAAC;IAED,6CAA6C;IAC7C,KAAK,CAAC,GAAG,CAAC,OAAuB,EAAE,MAAc,EAAE,MAAe;QAChE,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QACnC,IAAI,UAAU,IAAI,IAAI;YAAE,OAAO,OAAO,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;QAEzE,SAAS,aAAa,CAAC,UAA4B,EAAE,IAAoB;YACvE,OAAO,CAAC,GAAkB,EAAwB,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QACnF,CAAC;QAED,KAAK,IAAI,CAAC,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE;YAAE,OAAO,GAAG,aAAa,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;QACjG,OAAO,OAAO,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IACnD,CAAC;CACF"}

package/package.json

@@ -0,0 +1,30 @@
+{
+  "name": "@chunkd/source",
+  "version": "11.0.0",
+  "type": "module",
+  "engines": {
+    "node": ">=16.0.0"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/blacha/chunkd.git",
+    "directory": "packages/source"
+  },
+  "main": "./build/src/index.js",
+  "types": "./build/src/index.d.ts",
+  "author": "Blayne Chard",
+  "license": "MIT",
+  "scripts": {
+    "test": "node --test"
+  },
+  "devDependencies": {
+    "@types/node": "^18.0.0"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "files": [
+    "build/src"
+  ],
+  "gitHead": "b8136f070ded8ec3649c6cdf729e69b40ceec324"
+}