@indigoai-us/hq-cloud 5.19.1 → 5.21.0

package/src/sources/internals.ts

@@ -0,0 +1,46 @@
+/**
+ * Internal helpers shared between sources/list.ts and sources/get.ts.
+ *
+ * The S3 client factory hook exists so tests can inject an in-memory stub
+ * without standing up vi.mock for @aws-sdk/client-s3. Production code never
+ * calls the setter.
+ */
+
+import { S3Client } from "@aws-sdk/client-s3";
+import type { EntityContext } from "../types.js";
+
+function defaultFactory(ctx: EntityContext): S3Client {
+  return new S3Client({
+    region: ctx.region,
+    credentials: {
+      accessKeyId: ctx.credentials.accessKeyId,
+      secretAccessKey: ctx.credentials.secretAccessKey,
+      sessionToken: ctx.credentials.sessionToken,
+    },
+  });
+}
+
+let factory: (ctx: EntityContext) => S3Client = defaultFactory;
+
+export function getS3Client(ctx: EntityContext): S3Client {
+  return factory(ctx);
+}
+
+/** Test hook: replace the S3 client factory. */
+export function _setSourcesS3Factory(f: (ctx: EntityContext) => S3Client): void {
+  factory = f;
+}
+
+/** Test hook: restore the default S3 client factory. */
+export function _resetSourcesS3Factory(): void {
+  factory = defaultFactory;
+}
+
+export async function streamToString(body: unknown): Promise<string> {
+  const stream = body as AsyncIterable<Uint8Array>;
+  const chunks: Buffer[] = [];
+  for await (const chunk of stream) {
+    chunks.push(Buffer.from(chunk));
+  }
+  return Buffer.concat(chunks).toString("utf-8");
+}

package/src/sources/list.test.ts

@@ -0,0 +1,247 @@
+/**
+ * Unit tests for sources/list.ts.
+ *
+ * Uses an in-memory S3Client stub installed via the internal `_setSourcesS3Factory`
+ * hook so we don't have to stand up vi.mock for @aws-sdk/client-s3.
+ */
+
+import { describe, it, expect, beforeEach, afterEach } from "vitest";
+import {
+  GetObjectCommand,
+  ListObjectsV2Command,
+  type S3Client,
+} from "@aws-sdk/client-s3";
+import { listSources } from "./list.js";
+import {
+  _setSourcesS3Factory,
+  _resetSourcesS3Factory,
+} from "./internals.js";
+import type { EntityContext } from "../types.js";
+import { InvalidSourceChannelError } from "../schemas/source-channels.js";
+
+// ---------------------------------------------------------------------------
+// Fixtures + helpers
+// ---------------------------------------------------------------------------
+
+const ENTITY: EntityContext = {
+  uid: "cmp_test_001",
+  slug: "test",
+  bucketName: "hq-vault-test",
+  region: "us-east-1",
+  credentials: {
+    accessKeyId: "ASIATEST",
+    secretAccessKey: "secret",
+    sessionToken: "session",
+  },
+  expiresAt: new Date(Date.now() + 3600_000).toISOString(),
+};
+
+const MEETING_MD = `---
+source_id: abc
+source_type: meeting
+channel: meeting
+title: Hello
+---
+
+Body text.
+`;
+
+interface StoredObject {
+  key: string;
+  content: string;
+  lastModified: Date;
+}
+
+/** Build an in-memory S3 stub. Pagination is keyed by ContinuationToken (string offset). */
+function buildStub(
+  objects: StoredObject[],
+  observed: { commands: unknown[] } = { commands: [] },
+): S3Client {
+  async function send(command: unknown): Promise<unknown> {
+    observed.commands.push(command);
+
+    if (command instanceof ListObjectsV2Command) {
+      const { Prefix, MaxKeys, ContinuationToken } = command.input as {
+        Prefix?: string;
+        MaxKeys?: number;
+        ContinuationToken?: string;
+      };
+      const filtered = objects.filter((o) => !Prefix || o.key.startsWith(Prefix));
+      const offset = ContinuationToken ? parseInt(ContinuationToken, 10) : 0;
+      const max = MaxKeys ?? 1000;
+      const page = filtered.slice(offset, offset + max);
+      const nextOffset = offset + page.length;
+      const truncated = nextOffset < filtered.length;
+      return {
+        Contents: page.map((o) => ({
+          Key: o.key,
+          Size: Buffer.byteLength(o.content, "utf-8"),
+          LastModified: o.lastModified,
+          ETag: '"mock"',
+        })),
+        IsTruncated: truncated,
+        NextContinuationToken: truncated ? String(nextOffset) : undefined,
+        $metadata: {},
+      };
+    }
+
+    if (command instanceof GetObjectCommand) {
+      const { Key } = command.input as { Key: string };
+      const found = objects.find((o) => o.key === Key);
+      if (!found) {
+        const err = new Error(`NoSuchKey: ${Key}`);
+        (err as Error & { name: string }).name = "NoSuchKey";
+        throw err;
+      }
+      const content = found.content;
+      async function* stream(): AsyncIterable<Uint8Array> {
+        yield Buffer.from(content, "utf-8");
+      }
+      return { Body: stream(), $metadata: {} };
+    }
+
+    throw new Error(`unhandled command: ${(command as { constructor?: { name?: string } }).constructor?.name}`);
+  }
+  return { send } as unknown as S3Client;
+}
+
+function installStub(objects: StoredObject[]): { commands: unknown[] } {
+  const observed = { commands: [] as unknown[] };
+  const stub = buildStub(objects, observed);
+  _setSourcesS3Factory(() => stub);
+  return observed;
+}
+
+beforeEach(() => {
+  _resetSourcesS3Factory();
+});
+
+afterEach(() => {
+  _resetSourcesS3Factory();
+});
+
+// ---------------------------------------------------------------------------
+// Tests
+// ---------------------------------------------------------------------------
+
+describe("listSources", () => {
+  it("happy path: lists one source and skips its .raw.json sibling", async () => {
+    installStub([
+      {
+        key: "sources/meeting/abc.md",
+        content: MEETING_MD,
+        lastModified: new Date("2026-03-15T14:00:00Z"),
+      },
+      {
+        key: "sources/meeting/abc.raw.json",
+        content: '{"raw": true}',
+        lastModified: new Date("2026-03-15T14:00:00Z"),
+      },
+    ]);
+
+    const result = await listSources({ entity: ENTITY, channel: "meeting" });
+
+    expect(result.entries).toHaveLength(1);
+    expect(result.entries[0].sourceId).toBe("abc");
+    expect(result.entries[0].channel).toBe("meeting");
+    expect(result.entries[0].key).toBe("sources/meeting/abc.md");
+    expect(result.entries[0].size).toBeGreaterThan(0);
+    expect(result.entries[0].frontmatter).toBeUndefined();
+    expect(result.nextToken).toBeUndefined();
+  });
+
+  it("pagination: returns nextToken and accepts it on the next call", async () => {
+    const objects: StoredObject[] = Array.from({ length: 5 }, (_, i) => ({
+      key: `sources/meeting/m${i}.md`,
+      content: MEETING_MD,
+      lastModified: new Date("2026-03-15T14:00:00Z"),
+    }));
+    installStub(objects);
+
+    const page1 = await listSources({
+      entity: ENTITY,
+      channel: "meeting",
+      limit: 2,
+    });
+    expect(page1.entries).toHaveLength(2);
+    expect(page1.entries.map((e) => e.sourceId)).toEqual(["m0", "m1"]);
+    expect(page1.nextToken).toBe("2");
+
+    const page2 = await listSources({
+      entity: ENTITY,
+      channel: "meeting",
+      limit: 2,
+      continuationToken: page1.nextToken,
+    });
+    expect(page2.entries.map((e) => e.sourceId)).toEqual(["m2", "m3"]);
+    expect(page2.nextToken).toBe("4");
+
+    const page3 = await listSources({
+      entity: ENTITY,
+      channel: "meeting",
+      limit: 2,
+      continuationToken: page2.nextToken,
+    });
+    expect(page3.entries.map((e) => e.sourceId)).toEqual(["m4"]);
+    expect(page3.nextToken).toBeUndefined();
+  });
+
+  it("includeFrontmatter:true fetches each object and parses YAML", async () => {
+    const observed = installStub([
+      {
+        key: "sources/meeting/abc.md",
+        content: MEETING_MD,
+        lastModified: new Date("2026-03-15T14:00:00Z"),
+      },
+    ]);
+
+    const result = await listSources({
+      entity: ENTITY,
+      channel: "meeting",
+      includeFrontmatter: true,
+    });
+
+    expect(result.entries).toHaveLength(1);
+    expect(result.entries[0].frontmatter).toEqual({
+      source_id: "abc",
+      source_type: "meeting",
+      channel: "meeting",
+      title: "Hello",
+    });
+    // 1 list + 1 get for the single entry.
+    expect(observed.commands.filter((c) => c instanceof GetObjectCommand)).toHaveLength(1);
+  });
+
+  it("includeFrontmatter:false (default) does not perform GETs", async () => {
+    const observed = installStub([
+      {
+        key: "sources/meeting/abc.md",
+        content: MEETING_MD,
+        lastModified: new Date(),
+      },
+    ]);
+
+    await listSources({ entity: ENTITY, channel: "meeting" });
+
+    expect(observed.commands.filter((c) => c instanceof GetObjectCommand)).toHaveLength(0);
+  });
+
+  it("uses the correct prefix per channel", async () => {
+    const observed = installStub([]);
+    await listSources({ entity: ENTITY, channel: "slack" });
+    const listCmd = observed.commands[0] as ListObjectsV2Command;
+    expect((listCmd.input as { Prefix?: string }).Prefix).toBe("sources/slack/");
+  });
+
+  it("rejects an invalid channel via assertSourceChannel BEFORE any S3 call", async () => {
+    const observed = installStub([]);
+    await expect(
+      listSources({
+        entity: ENTITY,
+        // @ts-expect-error — invalid channel intentional for negative test
+        channel: "pigeon",
+      }),
+    ).rejects.toBeInstanceOf(InvalidSourceChannelError);
+    expect(observed.commands).toHaveLength(0);
+  });
+});

package/src/sources/list.ts

@@ -0,0 +1,95 @@
+/**
+ * listSources — page through `sources/{channel}/` keys in an entity bucket
+ * and return typed summaries.
+ *
+ * The `.raw.json` siblings written by the sources-pipeline are skipped from
+ * the summary (callers fetch them via getSource({ includeRaw: true })).
+ */
+
+import {
+  GetObjectCommand,
+  ListObjectsV2Command,
+} from "@aws-sdk/client-s3";
+import { assertSourceChannel } from "../schemas/source-channels.js";
+import { getS3Client, streamToString } from "./internals.js";
+import { parseFrontmatter } from "./parse.js";
+import type {
+  ListSourcesOptions,
+  ListSourcesResult,
+  SourceSummary,
+} from "./types.js";
+
+/**
+ * Strip the `sources/{channel}/` prefix and `.md` / `.raw.json` suffix from a key.
+ * Returns `null` if the key shape isn't recognised; the `isRawSibling` flag tells
+ * callers to skip pipeline-internal `.raw.json` companions from summaries.
+ */
+function deriveSourceId(
+  key: string,
+  prefix: string,
+): { sourceId: string; isRawSibling: boolean } | null {
+  if (!key.startsWith(prefix)) return null;
+  const tail = key.slice(prefix.length);
+  if (tail.endsWith(".raw.json")) {
+    return { sourceId: tail.slice(0, -".raw.json".length), isRawSibling: true };
+  }
+  if (tail.endsWith(".md")) {
+    return { sourceId: tail.slice(0, -".md".length), isRawSibling: false };
+  }
+  return null;
+}
+
+export async function listSources(opts: ListSourcesOptions): Promise<ListSourcesResult> {
+  // Validate channel BEFORE any S3 call (acceptance criterion).
+  assertSourceChannel(opts.channel);
+
+  const client = getS3Client(opts.entity);
+  const prefix = `sources/${opts.channel}/`;
+
+  const response = await client.send(
+    new ListObjectsV2Command({
+      Bucket: opts.entity.bucketName,
+      Prefix: prefix,
+      MaxKeys: opts.limit,
+      ContinuationToken: opts.continuationToken,
+    }),
+  );
+
+  const entries: SourceSummary[] = [];
+  for (const obj of response.Contents ?? []) {
+    if (!obj.Key) continue;
+    const derived = deriveSourceId(obj.Key, prefix);
+    if (!derived || derived.isRawSibling) continue;
+
+    const summary: SourceSummary = {
+      sourceId: derived.sourceId,
+      channel: opts.channel,
+      key: obj.Key,
+      lastModified: obj.LastModified ?? new Date(0),
+      size: obj.Size ?? 0,
+    };
+
+    if (opts.includeFrontmatter) {
+      try {
+        const get = await client.send(
+          new GetObjectCommand({
+            Bucket: opts.entity.bucketName,
+            Key: obj.Key,
+          }),
+        );
+        const text = await streamToString(get.Body);
+        const fm = parseFrontmatter(text);
+        if (fm) summary.frontmatter = fm;
+      } catch {
+        // Frontmatter fetch is best-effort; leave undefined on failure.
+      }
+    }
+
+    entries.push(summary);
+  }
+
+  return {
+    entries,
+    nextToken: response.IsTruncated ? response.NextContinuationToken : undefined,
+  };
+}

package/src/sources/parse.ts

@@ -0,0 +1,43 @@
+/**
+ * YAML frontmatter parsing for source markdown documents.
+ *
+ * Format: a leading `---\n` block terminated by `\n---\n`, followed by the body.
+ * Tolerates LF and CRLF. Returns `{ frontmatter: null, body }` when no
+ * frontmatter block is present.
+ */
+
+import { load as yamlLoad } from "js-yaml";
+
+export interface ParsedMarkdown {
+  frontmatter: Record<string, unknown> | null;
+  body: string;
+}
+
+const FRONTMATTER_RE = /^---\r?\n([\s\S]*?)\r?\n---\r?\n?([\s\S]*)$/;
+
+export function parseMarkdown(content: string): ParsedMarkdown {
+  const match = FRONTMATTER_RE.exec(content);
+  if (!match) {
+    return { frontmatter: null, body: content };
+  }
+
+  const [, yamlBlock, body] = match;
+  let frontmatter: Record<string, unknown> | null;
+  try {
+    const loaded = yamlLoad(yamlBlock);
+    frontmatter = loaded && typeof loaded === "object" && !Array.isArray(loaded)
+      ? (loaded as Record<string, unknown>)
+      : null;
+  } catch {
+    frontmatter = null;
+  }
+  return { frontmatter, body };
+}
+
+/**
+ * Convenience: parse only the frontmatter (used by listSources includeFrontmatter
+ * where we don't need the body).
+ */
+export function parseFrontmatter(content: string): Record<string, unknown> | null {
+  return parseMarkdown(content).frontmatter;
+}

package/src/sources/types.ts

@@ -0,0 +1,67 @@
+/**
+ * Public types for the sources read surface.
+ *
+ * Mirrors the sources-pipeline write path so a meeting source written by
+ * hq-pro is readable through this layer without translation.
+ */
+
+import type { SourceChannel } from "../schemas/source-channels.js";
+
+/**
+ * Lightweight summary returned by listSources(). `frontmatter` is only
+ * populated when the caller passes `includeFrontmatter: true`.
+ */
+export interface SourceSummary {
+  /** Stable ID derived from the object key (filename minus `.md`/`.raw.json`). */
+  sourceId: string;
+  channel: SourceChannel;
+  /** Full S3 object key (e.g. "sources/meeting/abc.md"). */
+  key: string;
+  lastModified: Date;
+  size: number;
+  /** Parsed YAML frontmatter — only present when listSources(includeFrontmatter:true). */
+  frontmatter?: Record<string, unknown>;
+}
+
+/**
+ * Full source document returned by getSource(). `frontmatter` is `null` if
+ * the document was malformed (no `---` delimiters) — body still contains the
+ * raw payload in that case so callers can fall back.
+ */
+export interface SourceDocument {
+  key: string;
+  /** Parsed YAML frontmatter, or `null` if the document had no frontmatter block. */
+  frontmatter: Record<string, unknown> | null;
+  /** Markdown body (everything after the closing `---`). */
+  body: string;
+  /** Sibling `.raw.json` payload, only present when getSource(includeRaw:true). */
+  raw?: unknown;
+}
+
+/**
+ * Listing options for listSources().
+ */
+export interface ListSourcesOptions {
+  entity: import("../types.js").EntityContext;
+  channel: SourceChannel;
+  /** Max keys per page; defaults to S3 default (1000). */
+  limit?: number;
+  /** Opaque continuation token returned by a prior page. */
+  continuationToken?: string;
+  /** When true, fetches+parses each entry's frontmatter (extra GETs). */
+  includeFrontmatter?: boolean;
+}
+
+export interface ListSourcesResult {
+  entries: SourceSummary[];
+  /** Present when the result was truncated; pass back as continuationToken. */
+  nextToken?: string;
+}
+
+export interface GetSourceOptions {
+  entity: import("../types.js").EntityContext;
+  channel: SourceChannel;
+  sourceId: string;
+  /** When true, also fetches the `.raw.json` sibling. */
+  includeRaw?: boolean;
+}