@hotmeshio/long-tail 0.1.0

package/build/services/storage/index.js

@@ -0,0 +1,31 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getStorageBackend = getStorageBackend;
+exports.resetStorageBackend = resetStorageBackend;
+let _backend = null;
+/**
+ * Returns the singleton storage backend.
+ *
+ * Backend selection (via LT_STORAGE_BACKEND env var):
+ *   "local" (default) — filesystem storage under LT_FILE_STORAGE_DIR
+ *   "s3"              — S3-compatible (MinIO, AWS S3, GCP Cloud Storage)
+ */
+function getStorageBackend() {
+    if (!_backend) {
+        const type = process.env.LT_STORAGE_BACKEND || 'local';
+        if (type === 's3') {
+            // Lazy require to avoid loading @aws-sdk when not needed
+            const { S3StorageBackend } = require('./s3');
+            _backend = new S3StorageBackend();
+        }
+        else {
+            const { LocalStorageBackend } = require('./local');
+            _backend = new LocalStorageBackend();
+        }
+    }
+    return _backend;
+}
+/** Reset the singleton (for testing). */
+function resetStorageBackend() {
+    _backend = null;
+}

package/build/services/storage/local.d.ts

@@ -0,0 +1,26 @@
+import type { StorageBackend } from './types';
+export declare class LocalStorageBackend implements StorageBackend {
+    write(key: string, data: Buffer): Promise<{
+        ref: string;
+        size: number;
+    }>;
+    read(key: string): Promise<{
+        data: Buffer;
+        size: number;
+    }>;
+    list(prefix?: string, pattern?: string): Promise<{
+        files: Array<{
+            path: string;
+            size: number;
+            modified_at: string;
+        }>;
+    }>;
+    delete(key: string): Promise<{
+        deleted: boolean;
+    }>;
+    getLocalPath(key: string): Promise<string>;
+    commitLocalPath(_key: string, _localPath: string): Promise<{
+        size: number;
+    }>;
+    createReadStream(key: string): Promise<NodeJS.ReadableStream>;
+}

package/build/services/storage/local.js

@@ -0,0 +1,129 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LocalStorageBackend = void 0;
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const BASE_DIR = process.env.LT_FILE_STORAGE_DIR || './data/files';
+function resolveAndValidate(filePath) {
+    const relative = filePath.replace(/^\/+/, '');
+    const resolved = path.resolve(BASE_DIR, relative);
+    const base = path.resolve(BASE_DIR);
+    if (!resolved.startsWith(base + path.sep) && resolved !== base) {
+        throw new Error(`Path traversal denied: ${filePath}`);
+    }
+    return resolved;
+}
+function ensureBaseDir() {
+    if (!fs.existsSync(BASE_DIR)) {
+        fs.mkdirSync(BASE_DIR, { recursive: true });
+    }
+}
+class LocalStorageBackend {
+    async write(key, data) {
+        ensureBaseDir();
+        const resolved = resolveAndValidate(key);
+        const dir = path.dirname(resolved);
+        if (!fs.existsSync(dir)) {
+            fs.mkdirSync(dir, { recursive: true });
+        }
+        fs.writeFileSync(resolved, data);
+        return { ref: key, size: data.length };
+    }
+    async read(key) {
+        const resolved = resolveAndValidate(key);
+        if (!fs.existsSync(resolved)) {
+            throw new Error(`File not found: ${key}`);
+        }
+        const data = fs.readFileSync(resolved);
+        return { data, size: data.length };
+    }
+    async list(prefix, pattern) {
+        ensureBaseDir();
+        const dir = prefix
+            ? resolveAndValidate(prefix)
+            : path.resolve(BASE_DIR);
+        if (!fs.existsSync(dir)) {
+            return { files: [] };
+        }
+        const entries = fs.readdirSync(dir, { withFileTypes: true });
+        const files = [];
+        for (const entry of entries) {
+            if (!entry.isFile())
+                continue;
+            if (pattern && !entry.name.match(new RegExp(pattern.replace(/\*/g, '.*'))))
+                continue;
+            const fullPath = path.join(dir, entry.name);
+            const stat = fs.statSync(fullPath);
+            const relativePath = path.relative(path.resolve(BASE_DIR), fullPath);
+            files.push({
+                path: relativePath,
+                size: stat.size,
+                modified_at: stat.mtime.toISOString(),
+            });
+        }
+        return { files };
+    }
+    async delete(key) {
+        const resolved = resolveAndValidate(key);
+        if (!fs.existsSync(resolved)) {
+            return { deleted: false };
+        }
+        fs.unlinkSync(resolved);
+        return { deleted: true };
+    }
+    async getLocalPath(key) {
+        ensureBaseDir();
+        const resolved = resolveAndValidate(key);
+        const dir = path.dirname(resolved);
+        if (!fs.existsSync(dir)) {
+            fs.mkdirSync(dir, { recursive: true });
+        }
+        return resolved;
+    }
+    async commitLocalPath(_key, _localPath) {
+        // No-op for local backend — file is already in its final location
+        const stat = fs.statSync(_localPath);
+        return { size: stat.size };
+    }
+    async createReadStream(key) {
+        const resolved = resolveAndValidate(key);
+        if (!fs.existsSync(resolved)) {
+            throw new Error(`File not found: ${key}`);
+        }
+        return fs.createReadStream(resolved);
+    }
+}
+exports.LocalStorageBackend = LocalStorageBackend;

package/build/services/storage/s3.d.ts

@@ -0,0 +1,31 @@
+import type { StorageBackend } from './types';
+export declare class S3StorageBackend implements StorageBackend {
+    private client;
+    private bucket;
+    private bucketReady;
+    constructor();
+    private ensureBucket;
+    write(key: string, data: Buffer): Promise<{
+        ref: string;
+        size: number;
+    }>;
+    read(key: string): Promise<{
+        data: Buffer;
+        size: number;
+    }>;
+    list(prefix?: string, pattern?: string): Promise<{
+        files: Array<{
+            path: string;
+            size: number;
+            modified_at: string;
+        }>;
+    }>;
+    delete(key: string): Promise<{
+        deleted: boolean;
+    }>;
+    getLocalPath(key: string): Promise<string>;
+    commitLocalPath(key: string, localPath: string): Promise<{
+        size: number;
+    }>;
+    createReadStream(key: string): Promise<NodeJS.ReadableStream>;
+}

package/build/services/storage/s3.js

@@ -0,0 +1,180 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.S3StorageBackend = void 0;
+const fs = __importStar(require("fs"));
+const os = __importStar(require("os"));
+const path = __importStar(require("path"));
+const client_s3_1 = require("@aws-sdk/client-s3");
+const STAGING_DIR = path.join(os.tmpdir(), 'lt-staging');
+class S3StorageBackend {
+    constructor() {
+        const endpoint = process.env.LT_S3_ENDPOINT;
+        const region = process.env.LT_S3_REGION || 'us-east-1';
+        const forcePathStyle = process.env.LT_S3_FORCE_PATH_STYLE === 'true';
+        this.bucket = process.env.LT_S3_BUCKET || 'long-tail-files';
+        const clientConfig = {
+            region,
+            forcePathStyle,
+        };
+        if (endpoint) {
+            clientConfig.endpoint = endpoint;
+        }
+        // Explicit credentials (MinIO, HMAC keys). If absent, SDK falls through
+        // to IAM role / instance metadata (standard AWS behavior).
+        const accessKey = process.env.LT_S3_ACCESS_KEY;
+        const secretKey = process.env.LT_S3_SECRET_KEY;
+        if (accessKey && secretKey) {
+            clientConfig.credentials = {
+                accessKeyId: accessKey,
+                secretAccessKey: secretKey,
+            };
+        }
+        this.client = new client_s3_1.S3Client(clientConfig);
+        this.bucketReady = this.ensureBucket();
+    }
+    async ensureBucket() {
+        try {
+            await this.client.send(new client_s3_1.HeadBucketCommand({ Bucket: this.bucket }));
+        }
+        catch {
+            try {
+                await this.client.send(new client_s3_1.CreateBucketCommand({ Bucket: this.bucket }));
+            }
+            catch (err) {
+                // Bucket may have been created concurrently
+                if (err.name !== 'BucketAlreadyOwnedByYou' && err.name !== 'BucketAlreadyExists') {
+                    throw err;
+                }
+            }
+        }
+    }
+    async write(key, data) {
+        await this.bucketReady;
+        await this.client.send(new client_s3_1.PutObjectCommand({
+            Bucket: this.bucket,
+            Key: normalizeKey(key),
+            Body: data,
+        }));
+        return { ref: key, size: data.length };
+    }
+    async read(key) {
+        await this.bucketReady;
+        const resp = await this.client.send(new client_s3_1.GetObjectCommand({
+            Bucket: this.bucket,
+            Key: normalizeKey(key),
+        }));
+        const chunks = [];
+        for await (const chunk of resp.Body) {
+            chunks.push(Buffer.from(chunk));
+        }
+        const data = Buffer.concat(chunks);
+        return { data, size: data.length };
+    }
+    async list(prefix, pattern) {
+        await this.bucketReady;
+        const normalizedPrefix = prefix ? normalizeKey(prefix) : undefined;
+        // Use Delimiter to mimic single-directory listing (like readdirSync)
+        const listPrefix = normalizedPrefix ? (normalizedPrefix.endsWith('/') ? normalizedPrefix : normalizedPrefix + '/') : undefined;
+        const resp = await this.client.send(new client_s3_1.ListObjectsV2Command({
+            Bucket: this.bucket,
+            Prefix: listPrefix,
+            Delimiter: '/',
+        }));
+        const files = [];
+        for (const obj of resp.Contents || []) {
+            if (!obj.Key || !obj.Size)
+                continue;
+            // Extract filename from key for pattern matching
+            const fileName = obj.Key.split('/').pop() || '';
+            if (pattern && !fileName.match(new RegExp(pattern.replace(/\*/g, '.*'))))
+                continue;
+            files.push({
+                path: obj.Key,
+                size: obj.Size,
+                modified_at: obj.LastModified?.toISOString() || new Date().toISOString(),
+            });
+        }
+        return { files };
+    }
+    async delete(key) {
+        await this.bucketReady;
+        try {
+            await this.client.send(new client_s3_1.DeleteObjectCommand({
+                Bucket: this.bucket,
+                Key: normalizeKey(key),
+            }));
+            return { deleted: true };
+        }
+        catch {
+            return { deleted: false };
+        }
+    }
+    async getLocalPath(key) {
+        const localPath = path.join(STAGING_DIR, normalizeKey(key));
+        const dir = path.dirname(localPath);
+        if (!fs.existsSync(dir)) {
+            fs.mkdirSync(dir, { recursive: true });
+        }
+        return localPath;
+    }
+    async commitLocalPath(key, localPath) {
+        const data = fs.readFileSync(localPath);
+        await this.write(key, data);
+        // Clean up staging file
+        try {
+            fs.unlinkSync(localPath);
+        }
+        catch { /* ignore */ }
+        return { size: data.length };
+    }
+    async createReadStream(key) {
+        await this.bucketReady;
+        const resp = await this.client.send(new client_s3_1.GetObjectCommand({
+            Bucket: this.bucket,
+            Key: normalizeKey(key),
+        }));
+        return resp.Body;
+    }
+}
+exports.S3StorageBackend = S3StorageBackend;
+/** Strip leading slashes to create a valid S3 key; reject traversal. */
+function normalizeKey(key) {
+    const normalized = key.replace(/^\/+/, '');
+    if (normalized.includes('..')) {
+        throw new Error(`Path traversal denied: ${key}`);
+    }
+    return normalized;
+}

package/build/services/storage/types.d.ts

@@ -0,0 +1,41 @@
+export interface StorageBackend {
+    /** Write data to storage. Returns the key and size. */
+    write(key: string, data: Buffer): Promise<{
+        ref: string;
+        size: number;
+    }>;
+    /** Read data from storage. Returns the raw buffer and size. */
+    read(key: string): Promise<{
+        data: Buffer;
+        size: number;
+    }>;
+    /** List files under a prefix/directory, optionally filtered by glob pattern. */
+    list(prefix?: string, pattern?: string): Promise<{
+        files: Array<{
+            path: string;
+            size: number;
+            modified_at: string;
+        }>;
+    }>;
+    /** Delete a file. Returns whether it existed. */
+    delete(key: string): Promise<{
+        deleted: boolean;
+    }>;
+    /**
+     * Get a local filesystem path for tools that need to write directly (e.g. Playwright).
+     * For local backend: returns the final storage path.
+     * For S3 backend: returns a temp staging path.
+     * Caller MUST call commitLocalPath() after writing to it.
+     */
+    getLocalPath(key: string): Promise<string>;
+    /**
+     * Commit a file written to a local path into storage.
+     * For local backend: no-op (file is already in place).
+     * For S3 backend: uploads the file, then removes the temp file.
+     */
+    commitLocalPath(key: string, localPath: string): Promise<{
+        size: number;
+    }>;
+    /** Create a readable stream for HTTP serving. */
+    createReadStream(key: string): Promise<NodeJS.ReadableStream>;
+}

package/build/services/storage/types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/build/services/task/crud.d.ts

@@ -0,0 +1,21 @@
+import type { LTTaskRecord, LTTaskStatus, LTMilestone } from '../../types';
+import type { CreateTaskInput, UpdateTaskInput } from './types';
+export declare function createTask(input: CreateTaskInput): Promise<LTTaskRecord>;
+export declare function updateTask(id: string, input: UpdateTaskInput): Promise<LTTaskRecord>;
+export declare function appendMilestones(id: string, milestones: LTMilestone[]): Promise<LTTaskRecord>;
+export declare function getTask(id: string): Promise<LTTaskRecord | null>;
+export declare function getTaskBySignalId(signalId: string): Promise<LTTaskRecord | null>;
+export declare function getTaskByWorkflowId(workflowId: string): Promise<LTTaskRecord | null>;
+export declare function listTasks(filters: {
+    status?: LTTaskStatus;
+    lt_type?: string;
+    workflow_type?: string;
+    workflow_id?: string;
+    parent_workflow_id?: string;
+    origin_id?: string;
+    limit?: number;
+    offset?: number;
+}): Promise<{
+    tasks: LTTaskRecord[];
+    total: number;
+}>;

package/build/services/task/crud.js

@@ -0,0 +1,123 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createTask = createTask;
+exports.updateTask = updateTask;
+exports.appendMilestones = appendMilestones;
+exports.getTask = getTask;
+exports.getTaskBySignalId = getTaskBySignalId;
+exports.getTaskByWorkflowId = getTaskByWorkflowId;
+exports.listTasks = listTasks;
+const db_1 = require("../db");
+const sql_1 = require("./sql");
+async function createTask(input) {
+    const pool = (0, db_1.getPool)();
+    const { rows } = await pool.query(sql_1.CREATE_TASK, [
+        input.workflow_id,
+        input.workflow_type,
+        input.lt_type,
+        input.task_queue || null,
+        input.signal_id,
+        input.parent_workflow_id,
+        input.origin_id || null,
+        input.parent_id || null,
+        input.envelope,
+        input.metadata ? JSON.stringify(input.metadata) : null,
+        input.priority || 2,
+        input.trace_id || null,
+        input.span_id || null,
+        input.initiated_by || null,
+        input.principal_type || null,
+        input.executing_as || null,
+    ]);
+    return rows[0];
+}
+async function updateTask(id, input) {
+    const pool = (0, db_1.getPool)();
+    const sets = [];
+    const values = [];
+    let idx = 1;
+    if (input.status !== undefined) {
+        sets.push(`status = $${idx++}`);
+        values.push(input.status);
+    }
+    if (input.completed_at !== undefined) {
+        sets.push(`completed_at = $${idx++}`);
+        values.push(input.completed_at);
+    }
+    if (input.error !== undefined) {
+        sets.push(`error = $${idx++}`);
+        values.push(input.error);
+    }
+    if (input.data !== undefined) {
+        sets.push(`data = $${idx++}`);
+        values.push(input.data);
+    }
+    if (input.milestones !== undefined) {
+        sets.push(`milestones = $${idx++}`);
+        values.push(JSON.stringify(input.milestones));
+    }
+    values.push(id);
+    const { rows } = await pool.query(`UPDATE lt_tasks SET ${sets.join(', ')} WHERE id = $${idx} RETURNING *`, values);
+    return rows[0];
+}
+async function appendMilestones(id, milestones) {
+    const pool = (0, db_1.getPool)();
+    const { rows } = await pool.query(sql_1.APPEND_MILESTONES, [JSON.stringify(milestones), id]);
+    return rows[0];
+}
+async function getTask(id) {
+    const pool = (0, db_1.getPool)();
+    const { rows } = await pool.query(sql_1.GET_TASK, [id]);
+    return rows[0] || null;
+}
+async function getTaskBySignalId(signalId) {
+    const pool = (0, db_1.getPool)();
+    const { rows } = await pool.query(sql_1.GET_TASK_BY_SIGNAL_ID, [signalId]);
+    return rows[0] || null;
+}
+async function getTaskByWorkflowId(workflowId) {
+    const pool = (0, db_1.getPool)();
+    const { rows } = await pool.query(sql_1.GET_TASK_BY_WORKFLOW_ID, [workflowId]);
+    return rows[0] || null;
+}
+async function listTasks(filters) {
+    const pool = (0, db_1.getPool)();
+    const conditions = [];
+    const values = [];
+    let idx = 1;
+    if (filters.status) {
+        conditions.push(`status = $${idx++}`);
+        values.push(filters.status);
+    }
+    if (filters.lt_type) {
+        conditions.push(`lt_type = $${idx++}`);
+        values.push(filters.lt_type);
+    }
+    if (filters.workflow_type) {
+        conditions.push(`workflow_type = $${idx++}`);
+        values.push(filters.workflow_type);
+    }
+    if (filters.workflow_id) {
+        conditions.push(`workflow_id = $${idx++}`);
+        values.push(filters.workflow_id);
+    }
+    if (filters.parent_workflow_id) {
+        conditions.push(`parent_workflow_id = $${idx++}`);
+        values.push(filters.parent_workflow_id);
+    }
+    if (filters.origin_id) {
+        conditions.push(`origin_id = $${idx++}`);
+        values.push(filters.origin_id);
+    }
+    const where = conditions.length ? `WHERE ${conditions.join(' AND ')}` : '';
+    const limit = filters.limit || 50;
+    const offset = filters.offset || 0;
+    const [countResult, dataResult] = await Promise.all([
+        pool.query(`SELECT COUNT(*) FROM lt_tasks ${where}`, values),
+        pool.query(`SELECT * FROM lt_tasks ${where} ORDER BY created_at DESC LIMIT $${idx++} OFFSET $${idx++}`, [...values, limit, offset]),
+    ]);
+    return {
+        tasks: dataResult.rows,
+        total: parseInt(countResult.rows[0].count, 10),
+    };
+}

package/build/services/task/index.d.ts

@@ -0,0 +1,4 @@
+export * from './types';
+export * from './crud';
+export * from './resolve';
+export * from './process';

package/build/services/task/index.js

@@ -0,0 +1,20 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./types"), exports);
+__exportStar(require("./crud"), exports);
+__exportStar(require("./resolve"), exports);
+__exportStar(require("./process"), exports);

package/build/services/task/process.d.ts

@@ -0,0 +1,14 @@
+import type { LTTaskRecord } from '../../types';
+import type { ProcessSummary, ProcessStats } from './types';
+export declare function listProcesses(filters: {
+    limit?: number;
+    offset?: number;
+    workflow_type?: string;
+    status?: string;
+    search?: string;
+}): Promise<{
+    processes: ProcessSummary[];
+    total: number;
+}>;
+export declare function getProcessStats(period?: string): Promise<ProcessStats>;
+export declare function getProcessTasks(originId: string): Promise<LTTaskRecord[]>;