@indigoai-us/hq-cli 5.345.142 → 5.345.144

package/dist/commands/library-add.js

@@ -0,0 +1,434 @@
+/**
+ * `hq library add|tag|ls` — put files into the company media library.
+ *
+ * `add` uploads each source (a local file or an http(s) URL) straight to the
+ * company vault offload prefix (`.hq-offload/blobs/<sha256>`) through
+ * `POST /v1/files/presign` (op "put"), then writes only `<dir>/<name>.hqcloud`
+ * into the tree: a `vault: company` placeholder with an `hq-asset` block. The
+ * bytes never land in the tree, and a URL source is never copied to disk: it
+ * is streamed once to hash it and once more into the vault.
+ *
+ * Presign wire contract (hq-pro files-presign):
+ *   body    { company: <cmp_uid>, keys: [{ key, op: "put", contentType,
+ *             metadata: { "hq-content-sha256": <hex> }, checksumSha256: <b64>,
+ *             ifNoneMatch: "*" }] }
+ *   200     { results: [{ key, op, url, headers? }
+ *                      | { key, op, error, code, refusalReason?, currentEtag?, upgradeUrl? }] }
+ *   402     PlanLimitReachedBody (blocked: "files.create") for the whole request
+ *
+ * Duplicates: the key is content-addressed, so a `content-identical` refusal
+ * or a 412 from the create-only PUT means the bytes are already in the vault
+ * and nothing is uploaded again. The PUT replays the presign `headers`
+ * verbatim, so S3 validates `x-amz-checksum-sha256` against the bytes it gets;
+ * the CLI also hashes what it streamed and checks the response checksum.
+ */
+import crypto from "node:crypto";
+import fs from "node:fs";
+import http from "node:http";
+import https from "node:https";
+import path from "node:path";
+import { Readable } from "node:stream";
+import { formatCloudAssetBlock, formatCloudPlaceholder, parseCloudPlaceholder, } from "@indigoai-us/hq-cloud";
+import { PLACEHOLDER_SUFFIX, companyOffloadKey, containedPath, durablePlaceholderFs, renderPointer, resolvePlaceholder, } from "./storage-offload.js";
+import { formatPlanGateError, formatUpgradeRefusal, planGateErrorFromResponse, } from "../utils/plan-gate-error.js";
+/** Metadata key hq-pro binds to `checksumSha256` (sync's content hash). */
+const CONTENT_SHA256_META_KEY = "hq-content-sha256";
+const MEDIA_TYPES = {
+    ".png": "image/png",
+    ".jpg": "image/jpeg",
+    ".jpeg": "image/jpeg",
+    ".gif": "image/gif",
+    ".webp": "image/webp",
+    ".svg": "image/svg+xml",
+    ".avif": "image/avif",
+    ".heic": "image/heic",
+    ".mp4": "video/mp4",
+    ".mov": "video/quicktime",
+    ".webm": "video/webm",
+    ".m4v": "video/x-m4v",
+    ".mp3": "audio/mpeg",
+    ".wav": "audio/wav",
+    ".pdf": "application/pdf",
+    ".zip": "application/zip",
+    ".json": "application/json",
+    ".txt": "text/plain",
+    ".md": "text/markdown",
+};
+export function contentTypeFor(name, served) {
+    const fromServer = served?.split(";")[0]?.trim().toLowerCase();
+    if (fromServer && fromServer !== "application/octet-stream")
+        return fromServer;
+    return MEDIA_TYPES[path.extname(name).toLowerCase()] ?? "application/octet-stream";
+}
+export function mediaKindFor(contentType) {
+    if (contentType.startsWith("image/"))
+        return "image";
+    if (contentType.startsWith("video/"))
+        return "video";
+    return "other";
+}
+/** A file name safe to place in one directory: no separators, no control chars. */
+export function safeAssetName(raw) {
+    const cleaned = raw
+        // eslint-disable-next-line no-control-regex -- strip C0 controls and DEL from a remote file name
+        .replace(/[\u0000-\u001f\u007f]/g, "")
+        .replace(/[\\/]/g, "_")
+        .trim();
+    if (!cleaned || cleaned === "." || cleaned === "..")
+        return "download";
+    return cleaned.endsWith(PLACEHOLDER_SUFFIX) ? cleaned.slice(0, -PLACEHOLDER_SUFFIX.length) || "download" : cleaned;
+}
+/** Classify an argument: an http(s) URL, otherwise a local file. */
+export function parseSource(input) {
+    if (/^[a-z][a-z0-9+.-]*:\/\//i.test(input)) {
+        let url;
+        try {
+            url = new URL(input);
+        }
+        catch {
+            throw new Error(`'${input}' is not a valid URL.`);
+        }
+        if (url.protocol !== "https:" && url.protocol !== "http:") {
+            throw new Error(`Only http and https URLs can be added ('${url.protocol}' given).`);
+        }
+        const last = url.pathname.split("/").filter(Boolean).pop() ?? "";
+        let decoded = last;
+        try {
+            decoded = decodeURIComponent(last);
+        }
+        catch {
+            // Keep the raw segment.
+        }
+        return { kind: "url", input, name: safeAssetName(decoded) };
+    }
+    return { kind: "file", input, name: safeAssetName(path.basename(input)) };
+}
+/** `companies/<slug>/...` HQ-relative POSIX path → slug, or null. */
+function companyOf(rel) {
+    return /^companies\/([^/]+)(?:\/|$)/.exec(rel)?.[1] ?? null;
+}
+function toPosix(p) {
+    return p.split(path.sep).join("/");
+}
+function hqRelative(hqRoot, abs) {
+    return toPosix(path.relative(fs.realpathSync(path.resolve(hqRoot)), abs));
+}
+/**
+ * The library directory, HQ-relative. `--to` resolves against the HQ root and
+ * must be inside `companies/<co>/`, so `hq storage fetch` can find the
+ * company vault from the placeholder path. The default is
+ * `companies/<co>/library`.
+ */
+export function resolveLibraryDir(hqRoot, options) {
+    if (options.to === undefined) {
+        if (!options.company)
+            throw new Error("Pass --company <slug> or --to companies/<slug>/<dir>.");
+        const dirRel = `companies/${options.company}/library`;
+        return { slug: options.company, dirRel, dirAbs: containedPath(hqRoot, dirRel) };
+    }
+    const dirAbs = containedPath(hqRoot, options.to);
+    const dirRel = hqRelative(hqRoot, dirAbs);
+    const slug = companyOf(dirRel);
+    if (!slug)
+        throw new Error(`--to must be inside companies/<slug>/ ('${options.to}' is not).`);
+    if (options.company && options.company !== slug) {
+        throw new Error(`--to is inside companies/${slug}/, but --company is ${options.company}.`);
+    }
+    return { slug, dirRel, dirAbs };
+}
+async function hashStream(stream) {
+    const hash = crypto.createHash("sha256");
+    let size = 0;
+    for await (const chunk of stream) {
+        const buf = chunk;
+        hash.update(buf);
+        size += buf.length;
+    }
+    return { size, sha256: hash.digest("hex") };
+}
+function sha256Base64(hex) {
+    return Buffer.from(hex, "hex").toString("base64");
+}
+export function offloadTimestamp(now) {
+    return now.toISOString().replace(/\.\d{3}Z$/, "Z");
+}
+function presignOutcome(row) {
+    const r = (row ?? {});
+    if (typeof r.url === "string") {
+        const headers = {};
+        if (r.headers && typeof r.headers === "object") {
+            for (const [k, v] of Object.entries(r.headers)) {
+                if (typeof v === "string")
+                    headers[k] = v;
+            }
+        }
+        return { kind: "upload", grant: { url: r.url, headers } };
+    }
+    if (r.refusalReason === "content-identical")
+        return { kind: "exists" };
+    const message = typeof r.error === "string" ? r.error : "The vault refused the upload.";
+    if (r.code === "PLAN_LIMIT_EXCEEDED") {
+        return {
+            kind: "refused",
+            error: formatUpgradeRefusal(message, typeof r.upgradeUrl === "string" ? r.upgradeUrl : undefined),
+        };
+    }
+    return { kind: "refused", error: message };
+}
+/**
+ * Stream one source into the vault and check what arrived: S3 validated the
+ * checksum header (when presign bound it), the bytes streamed hash to the
+ * digest, and any checksum S3 echoes matches. Returns false on 412 (the
+ * content-addressed key already exists).
+ */
+async function uploadVerified(source, digest, grant, deps) {
+    const { stream } = await deps.openSource(source);
+    const hash = crypto.createHash("sha256");
+    let sent = 0;
+    stream.on("data", (chunk) => {
+        hash.update(chunk);
+        sent += chunk.length;
+    });
+    const res = await deps.put(grant.url, grant.headers, stream, digest.size);
+    if (res.status === 412)
+        return false;
+    if (res.status < 200 || res.status >= 300) {
+        throw new Error(`Upload failed (HTTP ${res.status}). Nothing was written.`);
+    }
+    const echoed = res.headers["x-amz-checksum-sha256"];
+    if (sent !== digest.size ||
+        hash.digest("hex") !== digest.sha256 ||
+        (echoed !== undefined && echoed !== sha256Base64(digest.sha256))) {
+        throw new Error("The uploaded bytes changed while uploading; their sha256 does not match. Nothing was written.");
+    }
+    return true;
+}
+/**
+ * Testable core. Every source is hashed first, then one presign request
+ * covers all of them (hq-pro rate-limits presign per user). A 402 for the
+ * request refuses every source and writes nothing; a per-key refusal fails
+ * only that source.
+ */
+export async function runLibraryAdd(inputs, options, deps) {
+    if (inputs.length === 0)
+        throw new Error("Pass at least one file or URL.");
+    const { slug, dirRel, dirAbs } = resolveLibraryDir(deps.hqRoot, options);
+    const company = await deps.resolveCompany(slug);
+    const items = [];
+    const planned = [];
+    const claimed = new Set();
+    for (const input of inputs) {
+        const item = { source: input, ok: false };
+        items.push(item);
+        try {
+            const source = parseSource(input);
+            const originalRel = `${dirRel}/${source.name}`;
+            const originalAbs = path.join(dirAbs, source.name);
+            const placeholderAbs = `${originalAbs}${PLACEHOLDER_SUFFIX}`;
+            if (claimed.has(originalRel))
+                throw new Error(`Another source is also named '${source.name}'.`);
+            if (fs.existsSync(placeholderAbs))
+                throw new Error(`'${originalRel}${PLACEHOLDER_SUFFIX}' already exists.`);
+            if (fs.existsSync(originalAbs)) {
+                throw new Error(`'${originalRel}' already exists in the tree. Use \`hq storage offload\` for files already in HQ.`);
+            }
+            claimed.add(originalRel);
+            const opened = await deps.openSource(source);
+            const { size, sha256 } = await hashStream(opened.stream);
+            const digest = { size, sha256, contentType: contentTypeFor(source.name, opened.contentType) };
+            Object.assign(item, { originalPath: originalRel, size, sha256, key: companyOffloadKey(sha256) });
+            planned.push({ source, item, digest, placeholderAbs, originalRel });
+        }
+        catch (err) {
+            item.error = err.message;
+        }
+    }
+    if (planned.length === 0)
+        return { company: slug, items };
+    // One presign row per distinct sha256: identical sources share one key.
+    const bySha = new Map();
+    for (const p of planned)
+        if (!bySha.has(p.digest.sha256))
+            bySha.set(p.digest.sha256, p);
+    const keys = [...bySha.values()].map((p) => ({
+        key: companyOffloadKey(p.digest.sha256),
+        op: "put",
+        contentType: p.digest.contentType,
+        metadata: { [CONTENT_SHA256_META_KEY]: p.digest.sha256 },
+        checksumSha256: sha256Base64(p.digest.sha256),
+        ifNoneMatch: "*",
+    }));
+    const raw = await deps.presign({ company: company.uid, op: "put", keys });
+    const { error: planError, response } = await planGateErrorFromResponse(raw, company.uid);
+    if (planError) {
+        const message = formatPlanGateError(planError);
+        for (const p of planned)
+            p.item.error = message;
+        return { company: slug, items };
+    }
+    const body = (await response.json().catch(() => ({})));
+    if (!response.ok) {
+        const msg = typeof body.error === "string" ? body.error : `HTTP ${response.status}`;
+        for (const p of planned)
+            p.item.error = `Could not get an upload URL: ${msg}`;
+        return { company: slug, items };
+    }
+    const rows = Array.isArray(body.results) ? body.results : [];
+    const outcomes = new Map();
+    for (const row of rows) {
+        const key = row?.key;
+        if (typeof key === "string")
+            outcomes.set(key, presignOutcome(row));
+    }
+    const uploads = new Map();
+    for (const p of planned) {
+        try {
+            const key = companyOffloadKey(p.digest.sha256);
+            const outcome = outcomes.get(key);
+            if (!outcome)
+                throw new Error("The vault returned no upload URL for this file.");
+            if (outcome.kind === "refused")
+                throw new Error(outcome.error);
+            let uploaded = false;
+            if (outcome.kind === "upload") {
+                let pending = uploads.get(key);
+                const first = pending === undefined;
+                if (first) {
+                    pending = uploadVerified(p.source, p.digest, outcome.grant, deps);
+                    uploads.set(key, pending);
+                }
+                const didUpload = await pending;
+                uploaded = first && didUpload;
+            }
+            await writeAssetPlaceholder(p, options, deps, dirAbs);
+            Object.assign(p.item, { ok: true, uploaded, placeholderPath: `${p.originalRel}${PLACEHOLDER_SUFFIX}` });
+        }
+        catch (err) {
+            p.item.error = err.message;
+            continue;
+        }
+        const mediaKind = mediaKindFor(p.digest.contentType);
+        if (deps.caption && mediaKind !== "other") {
+            p.item.caption = await deps
+                .caption({
+                source: p.source,
+                placeholderAbs: p.placeholderAbs,
+                companyUid: company.uid,
+                contentType: p.digest.contentType,
+                mediaKind,
+                title: options.title ?? p.source.name,
+            })
+                .catch((err) => ({ status: "failed", note: err.message }));
+        }
+    }
+    return { company: slug, items };
+}
+async function writeAssetPlaceholder(p, options, deps, dirAbs) {
+    const ts = offloadTimestamp(deps.now());
+    const pointer = {
+        path: p.originalRel,
+        size: p.digest.size,
+        sha256: p.digest.sha256,
+        vault: "company",
+        key: companyOffloadKey(p.digest.sha256),
+        offloadedAt: ts,
+    };
+    const asset = {
+        title: options.title ?? p.source.name,
+        tags: options.tags,
+        ...(options.note ? { description: options.note, captionedBy: "person" } : {}),
+        media: { kind: mediaKindFor(p.digest.contentType) },
+        updatedAt: ts,
+    };
+    await fs.promises.mkdir(dirAbs, { recursive: true });
+    await deps.writePlaceholder(p.placeholderAbs, renderPointer(pointer, formatCloudAssetBlock(asset)));
+}
+/**
+ * Add and remove tags on a library placeholder. The placeholder is parsed and
+ * re-rendered by hq-cloud's placeholder writer, then replaced atomically
+ * (temp file + rename), so a sync watcher never reads a half-written file.
+ */
+export async function runLibraryTag(hqRoot, target, edits, now = () => new Date()) {
+    if (edits.add.length === 0 && edits.remove.length === 0)
+        throw new Error("Pass --add and/or --remove.");
+    const file = resolvePlaceholder(hqRoot, target);
+    const placeholderPath = hqRelative(hqRoot, file);
+    const placeholder = parseCloudPlaceholder(await fs.promises.readFile(file, "utf8"));
+    if (!placeholder || placeholder.vault !== "company") {
+        throw new Error(`'${placeholderPath}' is not a company library placeholder.`);
+    }
+    const before = placeholder.asset ?? { tags: [] };
+    const drop = new Set(edits.remove.map((tag) => tag.toLowerCase()));
+    const tags = before.tags.filter((tag) => !drop.has(tag.toLowerCase()));
+    for (const tag of edits.add) {
+        if (!tags.some((have) => have.toLowerCase() === tag.toLowerCase()))
+            tags.push(tag);
+    }
+    const changed = tags.length !== before.tags.length || tags.some((tag, i) => tag !== before.tags[i]);
+    if (!changed)
+        return { placeholderPath, tags, changed };
+    await replacePlaceholderText(file, formatCloudPlaceholder({
+        ...placeholder,
+        asset: { ...before, tags, updatedAt: offloadTimestamp(now()) },
+    }));
+    return { placeholderPath, tags, changed };
+}
+/** Replace a placeholder atomically (temp file + rename). */
+export async function replacePlaceholderText(file, text) {
+    const tmp = `${file}.${crypto.randomBytes(6).toString("hex")}.tmp`;
+    try {
+        await fs.promises.writeFile(tmp, text, { encoding: "utf8", flag: "wx" });
+        await fs.promises.rename(tmp, file);
+    }
+    finally {
+        await fs.promises.rm(tmp, { force: true });
+    }
+}
+// ── Default IO ──────────────────────────────────────────────────────────────
+/** Open a local file or GET a URL as a Node stream. */
+export async function openLibrarySource(source) {
+    if (source.kind === "file") {
+        const st = await fs.promises.stat(source.input);
+        if (!st.isFile())
+            throw new Error(`'${source.input}' is not a file.`);
+        return { stream: fs.createReadStream(source.input) };
+    }
+    const res = await fetch(source.input, { redirect: "follow" });
+    if (!res.ok || !res.body)
+        throw new Error(`Could not download ${source.input} (HTTP ${res.status}).`);
+    return {
+        stream: Readable.fromWeb(res.body),
+        contentType: res.headers.get("content-type") ?? undefined,
+    };
+}
+/**
+ * Stream a PUT with an explicit Content-Length: S3 presigned PUTs refuse
+ * chunked transfer encoding, which fetch would use for a stream body.
+ */
+export function streamPut(url, headers, body, size) {
+    return new Promise((resolve, reject) => {
+        const target = new URL(url);
+        const request = (target.protocol === "http:" ? http : https).request(target, { method: "PUT", headers: { ...headers, "content-length": String(size) } }, (res) => {
+            res.resume();
+            res.on("end", () => {
+                const out = {};
+                for (const [k, v] of Object.entries(res.headers))
+                    out[k] = Array.isArray(v) ? v.join(", ") : v;
+                resolve({ status: res.statusCode ?? 0, headers: out });
+            });
+            res.on("error", reject);
+        });
+        request.on("error", reject);
+        body.on("error", (err) => {
+            request.destroy(err);
+            reject(err);
+        });
+        body.pipe(request);
+    });
+}
+export const defaultLibraryIo = {
+    openSource: openLibrarySource,
+    put: streamPut,
+    writePlaceholder: durablePlaceholderFs.writePlaceholder,
+};
+//# sourceMappingURL=library-add.js.map

package/dist/commands/library-caption.d.ts

@@ -0,0 +1,112 @@
+import { type ProviderCaptureOptions, type ProviderCaptureResult } from "../lib/provider-launcher.js";
+import type { Readable } from "node:stream";
+import { type CloudAsset, type CloudAssetMediaKind } from "@indigoai-us/hq-cloud";
+import { type LibraryCompany, type LibrarySource, type OpenedSource } from "./library-add.js";
+/** Frame limits for a local caption run. */
+export declare const CAPTION_MAX_FRAMES = 4;
+export declare const CAPTION_MAX_FRAME_BYTES = 3750000;
+export declare const CAPTION_MAX_TOTAL_BYTES = 4000000;
+export declare const CAPTION_TIMEOUT_MS = 120000;
+export declare const LOCAL_CAPTION_PROVIDER = "claude";
+export type CaptionMediaKind = Exclude<CloudAssetMediaKind, "other">;
+export interface LibraryCaptionRequest {
+    source: LibrarySource;
+    placeholderAbs: string;
+    companyUid: string;
+    contentType: string;
+    mediaKind: CaptionMediaKind;
+    title?: string;
+}
+export interface LibraryCaptionNote {
+    status: "captioned" | "skipped" | "failed";
+    captionedBy?: string;
+    /** Plain-language reason when not captioned. */
+    note?: string;
+}
+export interface CaptionReply {
+    description: string;
+    tags: string[];
+    captionedBy: string;
+}
+export type CaptionOutcome = {
+    kind: "captioned";
+    reply: CaptionReply;
+} | {
+    kind: "skipped";
+    note: string;
+} | {
+    kind: "failed";
+    note: string;
+};
+export type RunLocalCaption = (args: readonly string[], options: ProviderCaptureOptions) => Promise<ProviderCaptureResult>;
+export declare const runLocalClaude: RunLocalCaption;
+export interface CaptionDeps {
+    /** Run the local Claude Code CLI headless (`claude -p ...`). */
+    runLocal: RunLocalCaption;
+    /** Frames (encoded image bytes) for a local file. */
+    extractFrames: (file: string, mediaKind: CaptionMediaKind) => Promise<Buffer[]>;
+}
+/** png / jpeg / gif / webp by magic bytes (formats Claude Code can read). */
+export declare function isCaptionableImage(bytes: Buffer): boolean;
+/** Keep frames in order while they fit the per-frame and total byte limits. */
+export declare function fitFrames(frames: readonly Buffer[]): Buffer[];
+/** Pull { description, tags } out of `claude -p --output-format json` output. */
+export declare function parseLocalCaption(stdout: string): {
+    description: string;
+    tags: string[];
+} | null;
+/** Map a local Claude Code run to an outcome, with one plain note when skipped. */
+export declare function captionOutcomeFromRun(run: ProviderCaptureResult): CaptionOutcome;
+/** Extract frames from `file` and caption them with local Claude Code. Never throws. */
+export declare function requestCaption(file: string, request: {
+    mediaKind: CaptionMediaKind;
+    title?: string;
+}, deps: CaptionDeps): Promise<CaptionOutcome>;
+/**
+ * Merge a caption into an asset. The user's tags stay first and suggested tags
+ * follow, deduplicated case-insensitively. A description a person wrote is
+ * kept unless `replaceDescription` is set (`hq library describe`).
+ */
+export declare function mergeCaption(asset: CloudAsset, reply: CaptionReply, options: {
+    replaceDescription: boolean;
+    updatedAt: string;
+}): CloudAsset;
+export interface AddCaptionDeps extends CaptionDeps {
+    now: () => Date;
+    openSource: (source: LibrarySource) => Promise<OpenedSource>;
+}
+/**
+ * Caption one asset `hq library add` just wrote. A URL source is read again
+ * into a temp file (it is never written into the tree); a local file is read
+ * in place. Never throws.
+ */
+export declare function captionAddedAsset(request: LibraryCaptionRequest, deps: AddCaptionDeps): Promise<LibraryCaptionNote>;
+export interface LibraryDescribeDeps extends CaptionDeps {
+    hqRoot: string;
+    now: () => Date;
+    resolveCompany: (slug: string) => Promise<LibraryCompany>;
+    /** POST /v1/files/presign. */
+    presign: (body: Record<string, unknown>) => Promise<Response>;
+    /** GET a presigned URL as a stream. */
+    download: (url: string) => Promise<Readable>;
+}
+export interface LibraryDescribeResult {
+    placeholderPath: string;
+    description: string;
+    tags: string[];
+    captionedBy: string;
+}
+/**
+ * `hq library describe <placeholder>`: download the asset, caption it again,
+ * and replace the description and captioned-by. Suggested tags are added to
+ * the existing tags. A caption that does not succeed is an error here.
+ */
+export declare function runLibraryDescribe(target: string, deps: LibraryDescribeDeps): Promise<LibraryDescribeResult>;
+export type RunTool = (command: string, args: string[]) => Promise<{
+    stdout: string;
+}>;
+/** Seek points at 10%, 35%, 60% and 85% of a known duration. */
+export declare function videoSeekPoints(durationSeconds: number | null): number[];
+/** Frames for captioning: the image itself when it fits, otherwise ffmpeg JPEGs. */
+export declare function extractCaptionFrames(file: string, mediaKind: CaptionMediaKind, run?: RunTool): Promise<Buffer[]>;
+//# sourceMappingURL=library-caption.d.ts.map