@pikku/core 0.12.137 → 0.12.139

package/dist/services/local-content-request-handler.js

@@ -1,37 +1,9 @@
 import { createReadStream } from 'fs';
-import { mkdir, stat, writeFile } from 'fs/promises';
-import { normalize, resolve } from 'path';
+import { stat } from 'fs/promises';
 import { Readable } from 'stream';
-import { signedContentPath } from './local-content.js';
+import { UploadTooLargeError, parseContentSizeLimit, resolveContentRequestTarget, signedContentPath, streamUploadToFile, } from './local-content.js';
 const matchesPrefix = (pathname, prefix) => pathname === prefix || pathname.startsWith(`${prefix}/`);
 const contentKey = (pathname, prefix) => pathname.slice(prefix.length).replace(/^\/+/, '');
-/**
- * Resolve a key against the content root, or `null` if it escapes. `normalize`
- * first so `..` segments are collapsed before the prefix check, and the
- * comparison carries a trailing separator so a sibling directory whose name
- * merely starts with the root's cannot pass as being inside it.
- */
-const toTargetPath = (basePath, key) => {
-    const normalizedBasePath = resolve(basePath);
-    const targetPath = resolve(normalizedBasePath, normalize(key));
-    return targetPath.startsWith(`${normalizedBasePath}/`) ? targetPath : null;
-};
-const parseSizeLimit = (sizeLimit) => {
-    const match = /^(\d+(?:\.\d+)?)(b|kb|mb|gb)?$/i.exec(sizeLimit.trim());
-    if (!match) {
-        throw new Error(`Invalid size limit: ${sizeLimit}`);
-    }
-    const value = Number(match[1]);
-    const unit = (match[2] ?? 'b').toLowerCase();
-    const multiplier = unit === 'gb'
-        ? 1024 * 1024 * 1024
-        : unit === 'mb'
-            ? 1024 * 1024
-            : unit === 'kb'
-                ? 1024
-                : 1;
-    return value * multiplier;
-};
 const text = (status, body) => new Response(body, {
     status,
     headers: { 'content-type': 'text/plain; charset=utf-8' },
@@ -112,47 +84,36 @@ export const createLocalContentRequestHandler = ({ content, logger, getJWT, }) =
             return text(signed.status, signed.body);
         }
         const key = contentKey(pathname, content.uploadUrlPrefix);
-        const targetPath = toTargetPath(content.localFileUploadPath, key);
-        if (!targetPath) {
+        const target = await resolveContentRequestTarget(content.localFileUploadPath, key, 'write');
+        if (!target) {
             return text(400, 'Invalid path');
         }
-        const maxBytes = parseSizeLimit(content.sizeLimit ?? '1mb');
-        // knowledge: decisions/security/an-upload-is-counted-as-it-arrives-not-buffered-then-measured.md
-        const chunks = [];
-        let bytesRead = 0;
-        const reader = request.body?.getReader();
-        if (reader) {
-            try {
-                for (;;) {
-                    const { done, value } = await reader.read();
-                    if (done)
-                        break;
-                    bytesRead += value.byteLength;
-                    if (bytesRead > maxBytes) {
-                        await reader.cancel();
-                        return text(413, 'Content too large');
-                    }
-                    chunks.push(Buffer.from(value));
-                }
-            }
-            finally {
-                reader.releaseLock();
+        try {
+            await streamUploadToFile(request.body
+                ? Readable.fromWeb(request.body)
+                : (async function* () { })(), target.path, parseContentSizeLimit(content.sizeLimit ?? '1mb'));
+        }
+        catch (error) {
+            if (error instanceof UploadTooLargeError) {
+                return text(413, 'Content too large');
             }
+            throw error;
         }
-        await mkdir(resolve(targetPath, '..'), { recursive: true });
-        await writeFile(targetPath, Buffer.concat(chunks));
         return new Response(null, { status: 200 });
     };
     const handleAsset = async (request, requestUrl, pathname) => {
         const key = contentKey(pathname, content.assetUrlPrefix);
-        const targetPath = toTargetPath(content.localFileUploadPath, key);
-        if (!targetPath) {
+        const target = await resolveContentRequestTarget(content.localFileUploadPath, key, 'read');
+        if (!target) {
             return text(400, 'Invalid path');
         }
-        const signed = await validateSignedAssetRequest(requestUrl);
-        if (!signed.ok) {
-            return text(signed.status, signed.body);
+        if (target.visibility === 'private') {
+            const signed = await validateSignedAssetRequest(requestUrl);
+            if (!signed.ok) {
+                return text(signed.status, signed.body);
+            }
         }
+        const targetPath = target.path;
         try {
             const file = await stat(targetPath);
             if (!file.isFile()) {

package/dist/services/local-content.d.ts

@@ -1,4 +1,4 @@
-import type { BucketKeyArgs, ContentService, CopyFileArgs, GetUploadURLArgs, JWTService, Logger, SignContentKeyArgs, SignURLArgs, UploadURLResult, WriteFileArgs } from '@pikku/core/services';
+import type { BucketKeyArgs, ContentService, ContentVisibility, GetDownloadURLArgs, CopyFileArgs, GetUploadURLArgs, JWTService, Logger, SignContentKeyArgs, SignURLArgs, UploadURLResult, WriteFileArgs } from '@pikku/core/services';
 export interface LocalContentConfig {
     localFileUploadPath: string;
     uploadUrlPrefix: string;
@@ -17,17 +17,59 @@ export interface LocalContentConfig {
  * rather than throwing, which keeps both sides in agreement.
  */
 export declare const signedContentPath: (urlOrPath: string) => string;
+export declare const PUBLIC_URL_SEGMENT = "_public";
+export declare const CONTENT_VISIBILITIES: readonly ContentVisibility[];
+export type ContentUrlPath = {
+    visibility: ContentVisibility;
+    rest: string;
+    prefixed: boolean;
+};
+/**
+ * Splits the part of a content URL after the asset/upload prefix into the
+ * visibility it names and the `<bucket>/<key>` remainder. Only a leading
+ * `_public/` segment names the public side; anything else is private.
+ * Private buckets may therefore not be named `_public`.
+ */
+export declare const parseContentUrlPath: (key: string) => ContentUrlPath;
+/**
+ * Resolves `<bucket>/<key>` against one side of the store. The side is a
+ * directory under the content root, and both the bucket and the key must stay
+ * inside it, so neither side can climb out or reach the other.
+ */
+export declare const resolveVisibilityPath: (basePath: string, visibility: ContentVisibility, bucket: string, key: string) => string | null;
+export type ContentRequestTarget = {
+    path: string;
+    visibility: ContentVisibility;
+};
+/**
+ * Maps the part of a content URL after the asset/upload prefix to a file under
+ * the content root: `_public/...` to the public side, anything else to the
+ * private side, falling back for reads to an unprefixed file from before the
+ * split. Returns `null` for a path that escapes its side or names no file.
+ * Every server of local content resolves through here, and skips the signature
+ * check only when the result is public.
+ */
+export declare const resolveContentRequestTarget: (basePath: string, key: string, mode: 'read' | 'write') => Promise<ContentRequestTarget | null>;
+export declare class UploadTooLargeError extends Error {
+    constructor();
+}
+export declare const parseContentSizeLimit: (sizeLimit: string) => number;
+export declare const streamUploadToFile: (source: AsyncIterable<Uint8Array>, targetPath: string, maxBytes: number) => Promise<number>;
 export declare class LocalContent implements ContentService {
     private config;
     private logger;
     private jwt;
     constructor(config: LocalContentConfig, logger: Logger, jwt: JWTService);
     private safePath;
+    private existingPath;
+    private legacyPath;
+    private publicBase;
     private joinKey;
     init(): Promise<void>;
     private signParams;
     signURL(args: SignURLArgs): Promise<string>;
     signContentKey(args: SignContentKeyArgs): Promise<string>;
+    getDownloadURL(args: GetDownloadURLArgs): Promise<string>;
     /**
      * Presigned upload, mirroring the read side: the URL carries a short-lived,
      * path-bound signature that {@link createLocalContentRequestHandler} verifies
@@ -40,5 +82,7 @@ export declare class LocalContent implements ContentService {
     readFile(args: BucketKeyArgs): Promise<ReadableStream | NodeJS.ReadableStream>;
     readFileAsBuffer(args: BucketKeyArgs): Promise<Buffer>;
     deleteFile(args: BucketKeyArgs): Promise<boolean>;
+    listFilesByPrefix(bucket: string, prefix: string, visibility?: ContentVisibility): Promise<string[]>;
+    deleteByPrefix(bucket: string, prefix: string, visibility?: ContentVisibility): Promise<number>;
     private createDirectoryForFile;
 }

package/dist/services/local-content.js

@@ -1,7 +1,9 @@
 import { createReadStream, createWriteStream, promises } from 'fs';
-import { mkdir, readFile } from 'fs/promises';
-import { resolve, normalize } from 'path';
+import { randomUUID } from 'crypto';
+import { mkdir, readFile, readdir, rename, rm, stat } from 'fs/promises';
+import { resolve, normalize, sep } from 'path';
 import { pipeline } from 'stream/promises';
+import { Transform } from 'stream';
 /**
  * The representation of an asset that gets bound into a signed URL's
  * signature, and that a server can reconstruct from an incoming request:
@@ -27,8 +29,122 @@ export const signedContentPath = (urlOrPath) => {
         return pathname;
     }
 };
+export const PUBLIC_URL_SEGMENT = '_public';
+export const CONTENT_VISIBILITIES = [
+    'private',
+    'public',
+];
+/**
+ * Splits the part of a content URL after the asset/upload prefix into the
+ * visibility it names and the `<bucket>/<key>` remainder. Only a leading
+ * `_public/` segment names the public side; anything else is private.
+ * Private buckets may therefore not be named `_public`.
+ */
+export const parseContentUrlPath = (key) => {
+    const [first, ...rest] = key.split('/');
+    if (first === PUBLIC_URL_SEGMENT) {
+        return { visibility: 'public', rest: rest.join('/'), prefixed: true };
+    }
+    return { visibility: 'private', rest: key, prefixed: false };
+};
+/**
+ * Resolves `<bucket>/<key>` against one side of the store. The side is a
+ * directory under the content root, and both the bucket and the key must stay
+ * inside it, so neither side can climb out or reach the other.
+ */
+export const resolveVisibilityPath = (basePath, visibility, bucket, key) => {
+    if (!CONTENT_VISIBILITIES.includes(visibility))
+        return null;
+    if (bucket.includes('\0') || key.includes('\0'))
+        return null;
+    if (visibility === 'private') {
+        const [first] = normalize(bucket).split(/[\\/]/);
+        if (first === PUBLIC_URL_SEGMENT)
+            return null;
+    }
+    const root = resolve(resolve(basePath), visibility);
+    const scoped = resolve(root, normalize(bucket));
+    const target = resolve(scoped, normalize(key));
+    const inside = (path) => path === root || path.startsWith(root + sep);
+    if (!inside(scoped) || !inside(target))
+        return null;
+    return target;
+};
+/**
+ * Maps the part of a content URL after the asset/upload prefix to a file under
+ * the content root: `_public/...` to the public side, anything else to the
+ * private side, falling back for reads to an unprefixed file from before the
+ * split. Returns `null` for a path that escapes its side or names no file.
+ * Every server of local content resolves through here, and skips the signature
+ * check only when the result is public.
+ */
+export const resolveContentRequestTarget = async (basePath, key, mode) => {
+    const parsed = parseContentUrlPath(key);
+    const root = resolve(resolve(basePath), parsed.visibility);
+    const path = resolveVisibilityPath(basePath, parsed.visibility, parsed.rest, '');
+    if (!path || path === root)
+        return null;
+    if (mode === 'read' && !parsed.prefixed) {
+        const exists = await stat(path).then(() => true, () => false);
+        if (!exists) {
+            const [first] = normalize(parsed.rest).split(/[\\/]/);
+            const base = resolve(basePath);
+            const legacy = resolve(base, normalize(parsed.rest));
+            if (first !== PUBLIC_URL_SEGMENT &&
+                first !== 'private' &&
+                legacy.startsWith(base + sep)) {
+                return { path: legacy, visibility: 'private' };
+            }
+        }
+    }
+    return { path, visibility: parsed.visibility };
+};
 /** How long a presigned upload URL stays valid. Short, like an S3 PUT URL. */
 const UPLOAD_URL_TTL_MS = 15 * 60 * 1000;
+export class UploadTooLargeError extends Error {
+    constructor() {
+        super('content_too_large');
+    }
+}
+export const parseContentSizeLimit = (sizeLimit) => {
+    const match = /^(\d+(?:\.\d+)?)(b|kb|mb|gb)?$/i.exec(sizeLimit.trim());
+    if (!match) {
+        throw new Error(`Invalid size limit: ${sizeLimit}`);
+    }
+    const unit = (match[2] ?? 'b').toLowerCase();
+    const multiplier = unit === 'gb'
+        ? 1024 * 1024 * 1024
+        : unit === 'mb'
+            ? 1024 * 1024
+            : unit === 'kb'
+                ? 1024
+                : 1;
+    return Number(match[1]) * multiplier;
+};
+export const streamUploadToFile = async (source, targetPath, maxBytes) => {
+    await mkdir(resolve(targetPath, '..'), { recursive: true });
+    const partial = `${targetPath}.${process.pid}.${Date.now()}.${randomUUID()}.part`;
+    let bytes = 0;
+    const counter = new Transform({
+        transform(chunk, _encoding, callback) {
+            bytes += chunk.length;
+            if (bytes > maxBytes) {
+                callback(new UploadTooLargeError());
+                return;
+            }
+            callback(null, chunk);
+        },
+    });
+    try {
+        await pipeline(source, counter, createWriteStream(partial));
+        await rename(partial, targetPath);
+        return bytes;
+    }
+    catch (error) {
+        await rm(partial, { force: true });
+        throw error;
+    }
+};
 export class LocalContent {
     config;
     logger;
@@ -41,15 +157,39 @@ export class LocalContent {
             throw new Error('LocalContent requires a JWTService: without one it cannot sign asset URLs, and unsigned URLs cannot be verified.');
         }
     }
-    safePath(bucket, key) {
-        const base = resolve(this.config.localFileUploadPath);
-        const scoped = resolve(base, normalize(bucket));
-        const target = resolve(scoped, normalize(key));
-        if (!target.startsWith(base + '/') && target !== base) {
+    safePath(bucket, key, visibility = 'private') {
+        const target = resolveVisibilityPath(this.config.localFileUploadPath, visibility, bucket, key);
+        if (!target) {
             throw new Error('Invalid asset key');
         }
         return target;
     }
+    async existingPath(bucket, key, visibility = 'private') {
+        const path = this.safePath(bucket, key, visibility);
+        if (visibility === 'private') {
+            const exists = await stat(path).then(() => true, () => false);
+            if (!exists) {
+                const legacy = this.legacyPath(bucket, key);
+                if (legacy &&
+                    (await stat(legacy).then(() => true, () => false))) {
+                    return legacy;
+                }
+            }
+        }
+        return path;
+    }
+    legacyPath(bucket, key) {
+        const [first] = normalize(bucket).split(/[\\/]/);
+        if (first === PUBLIC_URL_SEGMENT || first === 'private')
+            return null;
+        const base = resolve(this.config.localFileUploadPath);
+        const target = resolve(base, normalize(bucket), normalize(key));
+        return target.startsWith(base + sep) ? target : null;
+    }
+    publicBase() {
+        const origin = this.config.server ?? '';
+        return `${origin}${this.config.assetUrlPrefix}/${PUBLIC_URL_SEGMENT}`;
+    }
     joinKey(bucket, key) {
         return `${bucket}/${key}`;
     }
@@ -82,16 +222,27 @@ export class LocalContent {
         return `${args.url}?${params}`;
     }
     async signContentKey(args) {
+        const visibility = args.visibility ?? 'private';
+        this.safePath(args.bucket, args.contentKey, visibility);
         const fullKey = this.joinKey(args.bucket, args.contentKey);
-        const base = this.config.server
-            ? `${this.config.server}${this.config.assetUrlPrefix}/${fullKey}`
-            : `${this.config.assetUrlPrefix}/${fullKey}`;
+        const origin = this.config.server ?? '';
+        if (visibility === 'public') {
+            return `${this.publicBase()}/${fullKey}`;
+        }
         return this.signURL({
-            url: base,
+            url: `${origin}${this.config.assetUrlPrefix}/${fullKey}`,
             dateLessThan: args.dateLessThan,
             dateGreaterThan: args.dateGreaterThan,
         });
     }
+    async getDownloadURL(args) {
+        return this.signContentKey({
+            bucket: args.bucket,
+            contentKey: args.key,
+            visibility: args.visibility,
+            dateLessThan: new Date(Date.now() + (args.expiresInSeconds ?? 3600) * 1000),
+        });
+    }
     /**
      * Presigned upload, mirroring the read side: the URL carries a short-lived,
      * path-bound signature that {@link createLocalContentRequestHandler} verifies
@@ -99,11 +250,12 @@ export class LocalContent {
      * now refuses one — so this must sign, or every upload 403s.
      */
     async getUploadURL(args) {
-        void args.visibility;
+        const visibility = args.visibility ?? 'private';
+        this.safePath(args.bucket, args.fileKey, visibility);
         const fullKey = this.joinKey(args.bucket, args.fileKey);
         this.logger.debug(`Going to upload with key: ${fullKey}`);
         const uploadUrl = await this.signURL({
-            url: `${this.config.uploadUrlPrefix}/${fullKey}`,
+            url: `${this.config.uploadUrlPrefix}/${visibility === 'public' ? `${PUBLIC_URL_SEGMENT}/` : ''}${fullKey}`,
             dateLessThan: new Date(Date.now() + UPLOAD_URL_TTL_MS),
         });
         return {
@@ -114,7 +266,7 @@ export class LocalContent {
     }
     async writeFile(args) {
         this.logger.debug(`Writing file: ${args.bucket}/${args.key}`);
-        const path = this.safePath(args.bucket, args.key);
+        const path = this.safePath(args.bucket, args.key, args.visibility);
         try {
             await this.createDirectoryForFile(path);
             const fileStream = createWriteStream(path);
@@ -130,7 +282,7 @@ export class LocalContent {
     async copyFile(args) {
         this.logger.debug(`Writing file: ${args.bucket}/${args.key}`);
         try {
-            const path = this.safePath(args.bucket, args.key);
+            const path = this.safePath(args.bucket, args.key, args.visibility);
             await this.createDirectoryForFile(path);
             await promises.copyFile(args.fromAbsolutePath, path);
             return true;
@@ -143,7 +295,7 @@ export class LocalContent {
     }
     async readFile(args) {
         this.logger.debug(`Getting key: ${args.bucket}/${args.key}`);
-        const filePath = this.safePath(args.bucket, args.key);
+        const filePath = await this.existingPath(args.bucket, args.key, args.visibility);
         try {
             const stream = createReadStream(filePath);
             stream.on('error', (err) => {
@@ -157,14 +309,14 @@ export class LocalContent {
         }
     }
     async readFileAsBuffer(args) {
-        const filePath = this.safePath(args.bucket, args.key);
+        const filePath = await this.existingPath(args.bucket, args.key, args.visibility);
         this.logger.debug(`Reading file as buffer: ${args.bucket}/${args.key}`);
         return readFile(filePath);
     }
     async deleteFile(args) {
         this.logger.debug(`deleting key: ${args.bucket}/${args.key}`);
         try {
-            await promises.unlink(this.safePath(args.bucket, args.key));
+            await promises.unlink(this.safePath(args.bucket, args.key, args.visibility));
             return true;
         }
         catch (e) {
@@ -172,6 +324,38 @@ export class LocalContent {
         }
         return false;
     }
+    async listFilesByPrefix(bucket, prefix, visibility = 'private') {
+        const root = this.safePath(bucket, '', visibility);
+        const keys = [];
+        const walk = async (dir, rel) => {
+            let entries;
+            try {
+                entries = await readdir(dir, { withFileTypes: true });
+            }
+            catch {
+                return;
+            }
+            for (const entry of entries) {
+                const next = rel ? `${rel}/${entry.name}` : entry.name;
+                if (entry.isDirectory())
+                    await walk(resolve(dir, entry.name), next);
+                else if (entry.isFile() && next.startsWith(prefix))
+                    keys.push(next);
+            }
+        };
+        this.safePath(bucket, prefix, visibility);
+        await walk(root, '');
+        return keys.sort();
+    }
+    async deleteByPrefix(bucket, prefix, visibility = 'private') {
+        const keys = await this.listFilesByPrefix(bucket, prefix, visibility);
+        let deleted = 0;
+        for (const key of keys) {
+            if (await this.deleteFile({ bucket, key, visibility }))
+                deleted++;
+        }
+        return deleted;
+    }
     async createDirectoryForFile(path) {
         const dir = path.split('/').slice(0, -1).join('/');
         await mkdir(dir, { recursive: true });

package/dist/services/local-variables.d.ts

@@ -1,6 +1,11 @@
 import type { VariablesService } from './variables-service.js';
 export declare class LocalVariablesService implements VariablesService {
     private variables;
+    /**
+     * Defaults to `process.env`, resolved when the service is built rather than
+     * when this module loads, and guarded so a runtime without `process` gets an
+     * empty store instead of a ReferenceError.
+     */
     constructor(variables?: Record<string, string | undefined>);
     getAll(): Record<string, string | undefined>;
     getVariables<T extends Record<string, unknown> = Record<string, unknown>>(names: (keyof T & string)[]): Partial<T>;

package/dist/services/local-variables.js

@@ -1,7 +1,12 @@
 export class LocalVariablesService {
     variables;
-    constructor(variables = process.env) {
-        this.variables = variables;
+    /**
+     * Defaults to `process.env`, resolved when the service is built rather than
+     * when this module loads, and guarded so a runtime without `process` gets an
+     * empty store instead of a ReferenceError.
+     */
+    constructor(variables) {
+        this.variables = variables ?? globalThis.process?.env ?? {};
     }
     getAll() {
         return this.variables || {};

package/dist/services/queue-webhook-service.js

@@ -39,7 +39,7 @@ export class QueueWebhookService extends WebhookService {
         }
         if (secret) {
             const signatureHeader = webhookConfig?.signatureHeader ?? DEFAULT_WEBHOOK_SIGNATURE_HEADER;
-            headers[signatureHeader] = this.sign(secret, body);
+            headers[signatureHeader] = await this.sign(secret, body);
         }
         const jobData = {
             url: input.url,

package/dist/services/scoped-content-service.d.ts

@@ -0,0 +1,35 @@
+import type { BucketKeyArgs, ContentService, ContentVisibility, CopyFileArgs, GetDownloadURLArgs, GetUploadURLArgs, SignContentKeyArgs, SignURLArgs, UploadURLResult, WriteFileArgs } from './content-service.js';
+export type ContentGrantMode = 'read' | 'write';
+export declare const normaliseContentPath: (value: string) => string;
+/**
+ * A `ContentService` narrowed to one folder of the store.
+ *
+ * Whatever bucket the addon names lands under `<root>/<bucket>`, and the keys
+ * and buckets it passes are refused if they climb out. Starting a bucket with
+ * `@` names a path from the root of the store instead, which only works under
+ * a prefix the host granted: `read` allows reads and signed downloads, `write`
+ * adds uploads, writes and deletes, and the longest matching prefix decides.
+ * `signURL` and `copyFile` take a finished URL or a local filesystem path, so
+ * neither can be narrowed to a folder and both are refused.
+ */
+export declare class ScopedContentService implements ContentService {
+    private content;
+    private root;
+    private grants;
+    constructor(content: ContentService, root: string, grants?: Record<string, ContentGrantMode>);
+    private resolve;
+    private resolveClean;
+    signContentKey(args: SignContentKeyArgs): Promise<string>;
+    signURL(_args: SignURLArgs): Promise<string>;
+    getUploadURL(args: GetUploadURLArgs): Promise<UploadURLResult>;
+    deleteFile(args: BucketKeyArgs): Promise<boolean>;
+    writeFile(args: WriteFileArgs): Promise<boolean>;
+    copyFile(_args: CopyFileArgs): Promise<boolean>;
+    readFile(args: BucketKeyArgs): Promise<ReadableStream | NodeJS.ReadableStream>;
+    readFileAsBuffer(args: BucketKeyArgs): ReturnType<ContentService['readFileAsBuffer']>;
+    getDownloadURL(args: GetDownloadURLArgs): Promise<string>;
+    deleteByPrefix(bucket: string, prefix: string, visibility?: ContentVisibility): Promise<number>;
+    listFilesByPrefix(bucket: string, prefix: string, visibility?: ContentVisibility): Promise<string[]>;
+    private keysUnder;
+    private resolvePrefix;
+}