@indigoai-us/hq-cloud 6.15.0 → 6.15.2

package/src/sources/list.test.ts

@@ -1,402 +0,0 @@
-/**
- * Unit tests for sources/list.ts.
- *
- * Uses an in-memory S3Client stub installed via the internal `_setSourcesS3Factory`
- * hook so we don't have to stand up vi.mock for @aws-sdk/client-s3.
- */
-
-import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
-import { Readable } from "node:stream";
-import {
-  GetObjectCommand,
-  ListObjectsV2Command,
-  type S3Client,
-} from "@aws-sdk/client-s3";
-import { listSources } from "./list.js";
-import {
-  _setSourcesS3Factory,
-  _resetSourcesS3Factory,
-} from "./internals.js";
-import {
-  setPresignedGetTransportForTesting,
-  type PresignedGetTransport,
-  type PresignTransportClient,
-} from "../object-io.js";
-import type { VaultListedObject } from "../vault-client.js";
-import type { EntityContext } from "../types.js";
-import { InvalidSourceChannelError } from "../schemas/source-channels.js";
-
-// ---------------------------------------------------------------------------
-// Fixtures + helpers
-// ---------------------------------------------------------------------------
-
-const ENTITY: EntityContext = {
-  uid: "cmp_test_001",
-  slug: "test",
-  bucketName: "hq-vault-test",
-  region: "us-east-1",
-  credentials: {
-    accessKeyId: "ASIATEST",
-    secretAccessKey: "secret",
-    sessionToken: "session",
-  },
-  expiresAt: new Date(Date.now() + 3600_000).toISOString(),
-};
-
-const MEETING_MD = `---
-source_id: abc
-source_type: meeting
-channel: meeting
-title: Hello
----
-
-Body text.
-`;
-
-interface StoredObject {
-  key: string;
-  content: string;
-  lastModified: Date;
-}
-
-/** Build an in-memory S3 stub. Pagination is keyed by ContinuationToken (string offset). */
-function buildStub(
-  objects: StoredObject[],
-  observed: { commands: unknown[] } = { commands: [] },
-): S3Client {
-  async function send(command: unknown): Promise<unknown> {
-    observed.commands.push(command);
-
-    if (command instanceof ListObjectsV2Command) {
-      const { Prefix, MaxKeys, ContinuationToken } = command.input as {
-        Prefix?: string;
-        MaxKeys?: number;
-        ContinuationToken?: string;
-      };
-      const filtered = objects.filter((o) => !Prefix || o.key.startsWith(Prefix));
-      const offset = ContinuationToken ? parseInt(ContinuationToken, 10) : 0;
-      const max = MaxKeys ?? 1000;
-      const page = filtered.slice(offset, offset + max);
-      const nextOffset = offset + page.length;
-      const truncated = nextOffset < filtered.length;
-      return {
-        Contents: page.map((o) => ({
-          Key: o.key,
-          Size: Buffer.byteLength(o.content, "utf-8"),
-          LastModified: o.lastModified,
-          ETag: '"mock"',
-        })),
-        IsTruncated: truncated,
-        NextContinuationToken: truncated ? String(nextOffset) : undefined,
-        $metadata: {},
-      };
-    }
-
-    if (command instanceof GetObjectCommand) {
-      const { Key } = command.input as { Key: string };
-      const found = objects.find((o) => o.key === Key);
-      if (!found) {
-        const err = new Error(`NoSuchKey: ${Key}`);
-        (err as Error & { name: string }).name = "NoSuchKey";
-        throw err;
-      }
-      const content = found.content;
-      async function* stream(): AsyncIterable<Uint8Array> {
-        yield Buffer.from(content, "utf-8");
-      }
-      return { Body: stream(), $metadata: {} };
-    }
-
-    throw new Error(`unhandled command: ${(command as { constructor?: { name?: string } }).constructor?.name}`);
-  }
-  return { send } as unknown as S3Client;
-}
-
-function installStub(objects: StoredObject[]): { commands: unknown[] } {
-  const observed = { commands: [] as unknown[] };
-  const stub = buildStub(objects, observed);
-  _setSourcesS3Factory(() => stub);
-  return observed;
-}
-
-beforeEach(() => {
-  _resetSourcesS3Factory();
-});
-
-afterEach(() => {
-  _resetSourcesS3Factory();
-});
-
-// ---------------------------------------------------------------------------
-// Tests
-// ---------------------------------------------------------------------------
-
-describe("listSources", () => {
-  it("happy path: lists one source and skips its .raw.json sibling", async () => {
-    installStub([
-      {
-        key: "sources/meetings/abc.md",
-        content: MEETING_MD,
-        lastModified: new Date("2026-03-15T14:00:00Z"),
-      },
-      {
-        key: "sources/meetings/abc.raw.json",
-        content: '{"raw": true}',
-        lastModified: new Date("2026-03-15T14:00:00Z"),
-      },
-    ]);
-
-    const result = await listSources({ entity: ENTITY, channel: "meeting" });
-
-    expect(result.entries).toHaveLength(1);
-    expect(result.entries[0].sourceId).toBe("abc");
-    expect(result.entries[0].channel).toBe("meeting");
-    expect(result.entries[0].key).toBe("sources/meetings/abc.md");
-    expect(result.entries[0].size).toBeGreaterThan(0);
-    expect(result.entries[0].frontmatter).toBeUndefined();
-    expect(result.nextToken).toBeUndefined();
-  });
-
-  it("pagination: returns nextToken and accepts it on the next call", async () => {
-    const objects: StoredObject[] = Array.from({ length: 5 }, (_, i) => ({
-      key: `sources/meetings/m${i}.md`,
-      content: MEETING_MD,
-      lastModified: new Date("2026-03-15T14:00:00Z"),
-    }));
-    installStub(objects);
-
-    const page1 = await listSources({
-      entity: ENTITY,
-      channel: "meeting",
-      limit: 2,
-    });
-    expect(page1.entries).toHaveLength(2);
-    expect(page1.entries.map((e) => e.sourceId)).toEqual(["m0", "m1"]);
-    expect(page1.nextToken).toBe("2");
-
-    const page2 = await listSources({
-      entity: ENTITY,
-      channel: "meeting",
-      limit: 2,
-      continuationToken: page1.nextToken,
-    });
-    expect(page2.entries.map((e) => e.sourceId)).toEqual(["m2", "m3"]);
-    expect(page2.nextToken).toBe("4");
-
-    const page3 = await listSources({
-      entity: ENTITY,
-      channel: "meeting",
-      limit: 2,
-      continuationToken: page2.nextToken,
-    });
-    expect(page3.entries.map((e) => e.sourceId)).toEqual(["m4"]);
-    expect(page3.nextToken).toBeUndefined();
-  });
-
-  it("includeFrontmatter:true fetches each object and parses YAML", async () => {
-    const observed = installStub([
-      {
-        key: "sources/meetings/abc.md",
-        content: MEETING_MD,
-        lastModified: new Date("2026-03-15T14:00:00Z"),
-      },
-    ]);
-
-    const result = await listSources({
-      entity: ENTITY,
-      channel: "meeting",
-      includeFrontmatter: true,
-    });
-
-    expect(result.entries).toHaveLength(1);
-    expect(result.entries[0].frontmatter).toEqual({
-      source_id: "abc",
-      source_type: "meeting",
-      channel: "meeting",
-      title: "Hello",
-    });
-    // 1 list + 1 get for the single entry.
-    expect(observed.commands.filter((c) => c instanceof GetObjectCommand)).toHaveLength(1);
-  });
-
-  it("includeFrontmatter:false (default) does not perform GETs", async () => {
-    const observed = installStub([
-      {
-        key: "sources/meetings/abc.md",
-        content: MEETING_MD,
-        lastModified: new Date(),
-      },
-    ]);
-
-    await listSources({ entity: ENTITY, channel: "meeting" });
-
-    expect(observed.commands.filter((c) => c instanceof GetObjectCommand)).toHaveLength(0);
-  });
-
-  it("uses the correct prefix per channel", async () => {
-    const observed = installStub([]);
-    await listSources({ entity: ENTITY, channel: "slack" });
-    const listCmd = observed.commands[0] as ListObjectsV2Command;
-    expect((listCmd.input as { Prefix?: string }).Prefix).toBe("sources/slack/");
-  });
-
-  it("rejects an invalid channel via assertSourceChannel BEFORE any S3 call", async () => {
-    const observed = installStub([]);
-    await expect(
-      listSources({
-        entity: ENTITY,
-        // @ts-expect-error — invalid channel intentional for negative test
-        channel: "pigeon",
-      }),
-    ).rejects.toBeInstanceOf(InvalidSourceChannelError);
-    expect(observed.commands).toHaveLength(0);
-  });
-});
-
-// ---------------------------------------------------------------------------
-// Presigned transport (vault client + company vault) — HQ-59
-// ---------------------------------------------------------------------------
-
-interface PresignVaultStub {
-  vault: PresignTransportClient;
-  listCalls: Array<{ prefix?: string; cursor?: string }>;
-}
-
-function fetchBackedPresignedGetTransport(): PresignedGetTransport {
-  return {
-    async get(url, headers) {
-      const response = await fetch(url, { method: "GET", headers });
-      const body = response.body ? Readable.fromWeb(response.body) : undefined;
-      return {
-        status: response.status,
-        headers: response.headers,
-        body,
-        destroy: () => body?.destroy(),
-      };
-    },
-  };
-}
-
-/**
- * Vault stub for the list path: `listFiles` returns the configured ACL-filtered
- * page (the server is the ACL boundary — only readable keys come back) + an
- * opaque cursor; `presign`+fetch serve per-entry frontmatter GETs.
- */
-function makePresignVault(
-  page: { objects: VaultListedObject[]; cursor: string | null },
-  contents: Record<string, string> = {},
-): PresignVaultStub {
-  const listCalls: Array<{ prefix?: string; cursor?: string }> = [];
-  vi.stubGlobal(
-    "fetch",
-    vi.fn(async (url: string) => {
-      const key = new URL(url).searchParams.get("key") ?? "";
-      const c = contents[key];
-      if (c === undefined) return new Response(null, { status: 404 });
-      return new Response(Buffer.from(c, "utf-8"), { status: 200 });
-    }),
-  );
-  setPresignedGetTransportForTesting(fetchBackedPresignedGetTransport());
-  const vault: PresignTransportClient = {
-    presign: async (input) => ({
-      results: input.keys.map((k) => ({
-        key: k.key,
-        op: k.op ?? input.op ?? "get",
-        url: `https://signed.example/?key=${encodeURIComponent(k.key)}`,
-      })),
-      expiresAt: "2099-01-01T00:00:00.000Z",
-    }),
-    listFiles: async (_company, prefix, cursor) => {
-      listCalls.push({ prefix, cursor });
-      return {
-        objects: page.objects,
-        cursor: page.cursor,
-        truncated: page.cursor != null,
-      };
-    },
-  };
-  return { vault, listCalls };
-}
-
-function listed(key: string): VaultListedObject {
-  return {
-    key,
-    size: 10,
-    lastModified: "2026-02-01T00:00:00.000Z",
-    etag: "etag",
-    permission: "read",
-  };
-}
-
-describe("listSources — presigned transport (vault + cmp_)", () => {
-  afterEach(() => {
-    setPresignedGetTransportForTesting(null);
-    vi.unstubAllGlobals();
-  });
-
-  it("lists via the ACL-filtered vault endpoint and never touches S3", async () => {
-    _setSourcesS3Factory(
-      () =>
-        ({
-          send: async () => {
-            throw new Error("S3 must not be used on the presign path");
-          },
-        }) as unknown as S3Client,
-    );
-    const { vault, listCalls } = makePresignVault({
-      objects: [listed("sources/meetings/a.md"), listed("sources/meetings/a.raw.json")],
-      cursor: null,
-    });
-
-    const res = await listSources({ entity: ENTITY, channel: "meeting", vault });
-
-    // .raw.json sibling is skipped; only the .md entry surfaces.
-    expect(res.entries.map((e) => e.sourceId)).toEqual(["a"]);
-    expect(res.entries[0].key).toBe("sources/meetings/a.md");
-    expect(res.nextToken).toBeUndefined();
-    // listFiles was called with the channel prefix (ACL filtering is server-side).
-    expect(listCalls[0].prefix).toBe("sources/meetings/");
-  });
-
-  it("surfaces the opaque vault cursor as nextToken (advisory limit)", async () => {
-    const { vault } = makePresignVault({
-      objects: [listed("sources/meetings/a.md")],
-      cursor: "OPAQUE_CURSOR",
-    });
-    const res = await listSources({
-      entity: ENTITY,
-      channel: "meeting",
-      limit: 1,
-      vault,
-    });
-    expect(res.nextToken).toBe("OPAQUE_CURSOR");
-  });
-
-  it("includeFrontmatter parses each entry via a presigned GET", async () => {
-    const { vault } = makePresignVault(
-      { objects: [listed("sources/meetings/a.md")], cursor: null },
-      { "sources/meetings/a.md": MEETING_MD },
-    );
-    const res = await listSources({
-      entity: ENTITY,
-      channel: "meeting",
-      includeFrontmatter: true,
-      vault,
-    });
-    expect(res.entries[0].frontmatter).toMatchObject({ title: "Hello" });
-  });
-
-  it("forwards the continuationToken as the vault cursor", async () => {
-    const { vault, listCalls } = makePresignVault({
-      objects: [],
-      cursor: null,
-    });
-    await listSources({
-      entity: ENTITY,
-      channel: "meeting",
-      continuationToken: "RESUME_HERE",
-      vault,
-    });
-    expect(listCalls[0].cursor).toBe("RESUME_HERE");
-  });
-});

package/src/sources/list.ts

@@ -1,84 +0,0 @@
-/**
- * listSources — page through `sources/{channel}/` keys in an entity bucket
- * and return typed summaries.
- *
- * The `.raw.json` siblings written by the sources-pipeline are skipped from
- * the summary (callers fetch them via getSource({ includeRaw: true })).
- */
-
-import { assertSourceChannel, sourcePathSegment } from "../schemas/source-channels.js";
-import { listObjects, readObjectText } from "./internals.js";
-import { parseFrontmatter } from "./parse.js";
-import type {
-  ListSourcesOptions,
-  ListSourcesResult,
-  SourceSummary,
-} from "./types.js";
-
-/**
- * Strip the `sources/{channel}/` prefix and `.md` / `.raw.json` suffix from a key.
- * Returns `null` if the key shape isn't recognised; the `isRawSibling` flag tells
- * callers to skip pipeline-internal `.raw.json` companions from summaries.
- */
-function deriveSourceId(
-  key: string,
-  prefix: string,
-): { sourceId: string; isRawSibling: boolean } | null {
-  if (!key.startsWith(prefix)) return null;
-  const tail = key.slice(prefix.length);
-  if (tail.endsWith(".raw.json")) {
-    return { sourceId: tail.slice(0, -".raw.json".length), isRawSibling: true };
-  }
-  if (tail.endsWith(".md")) {
-    return { sourceId: tail.slice(0, -".md".length), isRawSibling: false };
-  }
-  return null;
-}
-
-export async function listSources(opts: ListSourcesOptions): Promise<ListSourcesResult> {
-  // Validate channel BEFORE any read (acceptance criterion).
-  assertSourceChannel(opts.channel);
-
-  // Use the canonical path segment from the schema; channel "meeting" maps
-  // to the plural "meetings/" prefix the sources-pipeline writes to.
-  const prefix = `sources/${sourcePathSegment(opts.channel)}/`;
-
-  // Presign transport ACL-filters server-side and controls page size, so
-  // `limit` is advisory there; under direct S3 it maps to MaxKeys. Paginate via
-  // the returned nextToken regardless of transport.
-  const page = await listObjects(opts, prefix, {
-    limit: opts.limit,
-    continuationToken: opts.continuationToken,
-  });
-
-  const entries: SourceSummary[] = [];
-  for (const obj of page.contents) {
-    const derived = deriveSourceId(obj.key, prefix);
-    if (!derived || derived.isRawSibling) continue;
-
-    const summary: SourceSummary = {
-      sourceId: derived.sourceId,
-      channel: opts.channel,
-      key: obj.key,
-      lastModified: obj.lastModified,
-      size: obj.size,
-    };
-
-    if (opts.includeFrontmatter) {
-      try {
-        const text = await readObjectText(opts, obj.key);
-        const fm = parseFrontmatter(text);
-        if (fm) summary.frontmatter = fm;
-      } catch {
-        // Frontmatter fetch is best-effort; leave undefined on failure.
-      }
-    }
-
-    entries.push(summary);
-  }
-
-  return {
-    entries,
-    nextToken: page.nextToken,
-  };
-}

package/src/sources/parse.ts

@@ -1,43 +0,0 @@
-/**
- * YAML frontmatter parsing for source markdown documents.
- *
- * Format: a leading `---\n` block terminated by `\n---\n`, followed by the body.
- * Tolerates LF and CRLF. Returns `{ frontmatter: null, body }` when no
- * frontmatter block is present.
- */
-
-import { load as yamlLoad } from "js-yaml";
-
-export interface ParsedMarkdown {
-  frontmatter: Record<string, unknown> | null;
-  body: string;
-}
-
-const FRONTMATTER_RE = /^---\r?\n([\s\S]*?)\r?\n---\r?\n?([\s\S]*)$/;
-
-export function parseMarkdown(content: string): ParsedMarkdown {
-  const match = FRONTMATTER_RE.exec(content);
-  if (!match) {
-    return { frontmatter: null, body: content };
-  }
-
-  const [, yamlBlock, body] = match;
-  let frontmatter: Record<string, unknown> | null;
-  try {
-    const loaded = yamlLoad(yamlBlock);
-    frontmatter = loaded && typeof loaded === "object" && !Array.isArray(loaded)
-      ? (loaded as Record<string, unknown>)
-      : null;
-  } catch {
-    frontmatter = null;
-  }
-  return { frontmatter, body };
-}
-
-/**
- * Convenience: parse only the frontmatter (used by listSources includeFrontmatter
- * where we don't need the body).
- */
-export function parseFrontmatter(content: string): Record<string, unknown> | null {
-  return parseMarkdown(content).frontmatter;
-}

package/src/sources/types.ts

@@ -1,84 +0,0 @@
-/**
- * Public types for the sources read surface.
- *
- * Mirrors the sources-pipeline write path so a meeting source written by
- * hq-pro is readable through this layer without translation.
- */
-
-import type { SourceChannel } from "../schemas/source-channels.js";
-import type { PresignTransportClient } from "../object-io.js";
-
-/**
- * Lightweight summary returned by listSources(). `frontmatter` is only
- * populated when the caller passes `includeFrontmatter: true`.
- */
-export interface SourceSummary {
-  /** Stable ID derived from the object key (filename minus `.md`/`.raw.json`). */
-  sourceId: string;
-  channel: SourceChannel;
-  /** Full S3 object key (e.g. "sources/meeting/abc.md"). */
-  key: string;
-  lastModified: Date;
-  size: number;
-  /** Parsed YAML frontmatter — only present when listSources(includeFrontmatter:true). */
-  frontmatter?: Record<string, unknown>;
-}
-
-/**
- * Full source document returned by getSource(). `frontmatter` is `null` if
- * the document was malformed (no `---` delimiters) — body still contains the
- * raw payload in that case so callers can fall back.
- */
-export interface SourceDocument {
-  key: string;
-  /** Parsed YAML frontmatter, or `null` if the document had no frontmatter block. */
-  frontmatter: Record<string, unknown> | null;
-  /** Markdown body (everything after the closing `---`). */
-  body: string;
-  /** Sibling `.raw.json` payload, only present when getSource(includeRaw:true). */
-  raw?: unknown;
-}
-
-/**
- * Listing options for listSources().
- */
-export interface ListSourcesOptions {
-  entity: import("../types.js").EntityContext;
-  channel: SourceChannel;
-  /**
-   * Max keys per page. Under direct S3 this maps to `MaxKeys`; under the
-   * presigned transport ({@link vault} + a company vault) it is advisory — the
-   * vault `list` endpoint controls page size server-side. Defaults to the S3
-   * default (1000) on the direct path.
-   */
-  limit?: number;
-  /** Opaque continuation token returned by a prior page. */
-  continuationToken?: string;
-  /** When true, fetches+parses each entry's frontmatter (extra GETs). */
-  includeFrontmatter?: boolean;
-  /**
-   * Presign-capable vault client. When supplied and {@link entity} is a company
-   * vault (cmp_*), reads use the presigned-URL transport (ACL-filtered, no
-   * direct S3). Personal vaults and the absent-client path stay on direct S3.
-   */
-  vault?: PresignTransportClient;
-}
-
-export interface ListSourcesResult {
-  entries: SourceSummary[];
-  /** Present when the result was truncated; pass back as continuationToken. */
-  nextToken?: string;
-}
-
-export interface GetSourceOptions {
-  entity: import("../types.js").EntityContext;
-  channel: SourceChannel;
-  sourceId: string;
-  /** When true, also fetches the `.raw.json` sibling. */
-  includeRaw?: boolean;
-  /**
-   * Presign-capable vault client. When supplied and {@link entity} is a company
-   * vault (cmp_*), reads use the presigned-URL transport (no direct S3).
-   */
-  vault?: PresignTransportClient;
-}