@indigoai-us/hq-cloud 5.19.1 → 5.21.0

package/src/signals/list.test.ts

@@ -0,0 +1,283 @@
+/**
+ * Unit tests for signals/list.ts.
+ *
+ * Uses an in-memory S3Client stub installed via the internal
+ * `_setSignalsS3Factory` hook so we don't have to stand up vi.mock for
+ * @aws-sdk/client-s3.
+ */
+
+import { describe, it, expect, beforeEach, afterEach } from "vitest";
+import {
+  GetObjectCommand,
+  ListObjectsV2Command,
+  type S3Client,
+} from "@aws-sdk/client-s3";
+import { listSignals } from "./list.js";
+import {
+  _setSignalsS3Factory,
+  _resetSignalsS3Factory,
+} from "./internals.js";
+import type { EntityContext } from "../types.js";
+import { InvalidSignalTypeError, SIGNAL_TYPES } from "../schemas/signal-types.js";
+
+// ---------------------------------------------------------------------------
+// Fixtures + helpers
+// ---------------------------------------------------------------------------
+
+const ENTITY: EntityContext = {
+  uid: "cmp_test_001",
+  slug: "test",
+  bucketName: "hq-vault-test",
+  region: "us-east-1",
+  credentials: {
+    accessKeyId: "ASIATEST",
+    secretAccessKey: "secret",
+    sessionToken: "session",
+  },
+  expiresAt: new Date(Date.now() + 3600_000).toISOString(),
+};
+
+const ACTION_ITEM_MD = `---
+signal_id: sig_action_001
+signal_type: action_item
+source_ref: abc-meeting-001
+citations:
+  - "Stefan to review the sources pipeline implementation by end of week"
+entity_refs:
+  - stefan@indigoai.us
+---
+
+Stefan will review the sources pipeline implementation.
+`;
+
+interface StoredObject {
+  key: string;
+  content: string;
+  lastModified: Date;
+}
+
+function buildStub(
+  objects: StoredObject[],
+  observed: { commands: unknown[] } = { commands: [] },
+): S3Client {
+  async function send(command: unknown): Promise<unknown> {
+    observed.commands.push(command);
+
+    if (command instanceof ListObjectsV2Command) {
+      const { Prefix, MaxKeys, ContinuationToken } = command.input as {
+        Prefix?: string;
+        MaxKeys?: number;
+        ContinuationToken?: string;
+      };
+      const filtered = objects.filter((o) => !Prefix || o.key.startsWith(Prefix));
+      const offset = ContinuationToken ? parseInt(ContinuationToken, 10) : 0;
+      const max = MaxKeys ?? 1000;
+      const page = filtered.slice(offset, offset + max);
+      const nextOffset = offset + page.length;
+      const truncated = nextOffset < filtered.length;
+      return {
+        Contents: page.map((o) => ({
+          Key: o.key,
+          Size: Buffer.byteLength(o.content, "utf-8"),
+          LastModified: o.lastModified,
+          ETag: '"mock"',
+        })),
+        IsTruncated: truncated,
+        NextContinuationToken: truncated ? String(nextOffset) : undefined,
+        $metadata: {},
+      };
+    }
+
+    if (command instanceof GetObjectCommand) {
+      const { Key } = command.input as { Key: string };
+      const found = objects.find((o) => o.key === Key);
+      if (!found) {
+        const err = new Error(`NoSuchKey: ${Key}`);
+        (err as Error & { name: string }).name = "NoSuchKey";
+        throw err;
+      }
+      const content = found.content;
+      async function* stream(): AsyncIterable<Uint8Array> {
+        yield Buffer.from(content, "utf-8");
+      }
+      return { Body: stream(), $metadata: {} };
+    }
+
+    throw new Error(
+      `unhandled command: ${(command as { constructor?: { name?: string } }).constructor?.name}`,
+    );
+  }
+  return { send } as unknown as S3Client;
+}
+
+function installStub(objects: StoredObject[]): { commands: unknown[] } {
+  const observed = { commands: [] as unknown[] };
+  const stub = buildStub(objects, observed);
+  _setSignalsS3Factory(() => stub);
+  return observed;
+}
+
+beforeEach(() => {
+  _resetSignalsS3Factory();
+});
+
+afterEach(() => {
+  _resetSignalsS3Factory();
+});
+
+// ---------------------------------------------------------------------------
+// Tests
+// ---------------------------------------------------------------------------
+
+describe("listSignals", () => {
+  it("happy path: lists one signal under action_item", async () => {
+    installStub([
+      {
+        key: "signals/action_item/foo.md",
+        content: ACTION_ITEM_MD,
+        lastModified: new Date("2026-03-15T15:00:00Z"),
+      },
+    ]);
+
+    const result = await listSignals({ entity: ENTITY, signalType: "action_item" });
+
+    expect(result.entries).toHaveLength(1);
+    expect(result.entries[0].signalId).toBe("foo");
+    expect(result.entries[0].signalType).toBe("action_item");
+    expect(result.entries[0].key).toBe("signals/action_item/foo.md");
+    expect(result.entries[0].size).toBeGreaterThan(0);
+    expect(result.entries[0].frontmatter).toBeUndefined();
+    expect(result.entries[0].sourceRef).toBeUndefined();
+    expect(result.nextToken).toBeUndefined();
+  });
+
+  it("supports all six canonical signal types", async () => {
+    // Build one object per signal type, verify each is listable.
+    const objects: StoredObject[] = SIGNAL_TYPES.map((t) => ({
+      key: `signals/${t}/sig-${t}.md`,
+      content: ACTION_ITEM_MD,
+      lastModified: new Date("2026-03-15T15:00:00Z"),
+    }));
+
+    for (const signalType of SIGNAL_TYPES) {
+      installStub(objects);
+      const result = await listSignals({ entity: ENTITY, signalType });
+      expect(result.entries).toHaveLength(1);
+      expect(result.entries[0].signalId).toBe(`sig-${signalType}`);
+      expect(result.entries[0].signalType).toBe(signalType);
+    }
+  });
+
+  it("pagination: returns nextToken and accepts it on the next call", async () => {
+    const objects: StoredObject[] = Array.from({ length: 5 }, (_, i) => ({
+      key: `signals/action_item/s${i}.md`,
+      content: ACTION_ITEM_MD,
+      lastModified: new Date("2026-03-15T15:00:00Z"),
+    }));
+    installStub(objects);
+
+    const page1 = await listSignals({
+      entity: ENTITY,
+      signalType: "action_item",
+      limit: 2,
+    });
+    expect(page1.entries.map((e) => e.signalId)).toEqual(["s0", "s1"]);
+    expect(page1.nextToken).toBe("2");
+
+    const page2 = await listSignals({
+      entity: ENTITY,
+      signalType: "action_item",
+      limit: 2,
+      continuationToken: page1.nextToken,
+    });
+    expect(page2.entries.map((e) => e.signalId)).toEqual(["s2", "s3"]);
+    expect(page2.nextToken).toBe("4");
+
+    const page3 = await listSignals({
+      entity: ENTITY,
+      signalType: "action_item",
+      limit: 2,
+      continuationToken: page2.nextToken,
+    });
+    expect(page3.entries.map((e) => e.signalId)).toEqual(["s4"]);
+    expect(page3.nextToken).toBeUndefined();
+  });
+
+  it("includeFrontmatter:true fetches each object, parses YAML, surfaces sourceRef", async () => {
+    const observed = installStub([
+      {
+        key: "signals/action_item/foo.md",
+        content: ACTION_ITEM_MD,
+        lastModified: new Date("2026-03-15T15:00:00Z"),
+      },
+    ]);
+
+    const result = await listSignals({
+      entity: ENTITY,
+      signalType: "action_item",
+      includeFrontmatter: true,
+    });
+
+    expect(result.entries).toHaveLength(1);
+    expect(result.entries[0].sourceRef).toBe("abc-meeting-001");
+    expect(result.entries[0].frontmatter).toMatchObject({
+      signal_id: "sig_action_001",
+      signal_type: "action_item",
+      source_ref: "abc-meeting-001",
+    });
+    expect(observed.commands.filter((c) => c instanceof GetObjectCommand)).toHaveLength(1);
+  });
+
+  it("includeFrontmatter:true tolerates malformed frontmatter (sourceRef stays undefined)", async () => {
+    installStub([
+      {
+        key: "signals/action_item/bad.md",
+        content: "no frontmatter here, just body",
+        lastModified: new Date(),
+      },
+    ]);
+
+    const result = await listSignals({
+      entity: ENTITY,
+      signalType: "action_item",
+      includeFrontmatter: true,
+    });
+
+    expect(result.entries).toHaveLength(1);
+    expect(result.entries[0].sourceRef).toBeUndefined();
+    expect(result.entries[0].frontmatter).toBeUndefined();
+  });
+
+  it("includeFrontmatter:false (default) does not perform GETs", async () => {
+    const observed = installStub([
+      {
+        key: "signals/action_item/foo.md",
+        content: ACTION_ITEM_MD,
+        lastModified: new Date(),
+      },
+    ]);
+
+    await listSignals({ entity: ENTITY, signalType: "action_item" });
+
+    expect(observed.commands.filter((c) => c instanceof GetObjectCommand)).toHaveLength(0);
+  });
+
+  it("uses the correct prefix per signal type", async () => {
+    const observed = installStub([]);
+    await listSignals({ entity: ENTITY, signalType: "decision" });
+    const listCmd = observed.commands[0] as ListObjectsV2Command;
+    expect((listCmd.input as { Prefix?: string }).Prefix).toBe("signals/decision/");
+  });
+
+  it("rejects an invalid signalType via assertSignalType BEFORE any S3 call", async () => {
+    const observed = installStub([]);
+    await expect(
+      listSignals({
+        entity: ENTITY,
+        // @ts-expect-error — invalid signal type intentional for negative test
+        signalType: "ramble",
+      }),
+    ).rejects.toBeInstanceOf(InvalidSignalTypeError);
+    expect(observed.commands).toHaveLength(0);
+  });
+});

package/src/signals/list.ts

@@ -0,0 +1,92 @@
+/**
+ * listSignals — page through `signals/{signalType}/` keys in an entity bucket
+ * and return typed summaries.
+ *
+ * `sourceRef` is surfaced on each summary when the caller passes
+ * `includeFrontmatter: true` (otherwise the field is undefined).
+ */
+
+import {
+  GetObjectCommand,
+  ListObjectsV2Command,
+} from "@aws-sdk/client-s3";
+import { assertSignalType } from "../schemas/signal-types.js";
+import { getS3Client, streamToString } from "./internals.js";
+import { parseFrontmatter } from "./parse.js";
+import type {
+  ListSignalsOptions,
+  ListSignalsResult,
+  SignalSummary,
+} from "./types.js";
+
+/**
+ * Strip the `signals/{signalType}/` prefix and `.md` suffix from a key.
+ * Returns `null` if the key shape isn't recognised.
+ */
+function deriveSignalId(key: string, prefix: string): string | null {
+  if (!key.startsWith(prefix)) return null;
+  const tail = key.slice(prefix.length);
+  if (!tail.endsWith(".md")) return null;
+  return tail.slice(0, -".md".length);
+}
+
+export async function listSignals(opts: ListSignalsOptions): Promise<ListSignalsResult> {
+  // Validate signalType BEFORE any S3 call (acceptance criterion + agent-mitigation).
+  assertSignalType(opts.signalType);
+
+  const client = getS3Client(opts.entity);
+  const prefix = `signals/${opts.signalType}/`;
+
+  const response = await client.send(
+    new ListObjectsV2Command({
+      Bucket: opts.entity.bucketName,
+      Prefix: prefix,
+      MaxKeys: opts.limit,
+      ContinuationToken: opts.continuationToken,
+    }),
+  );
+
+  const entries: SignalSummary[] = [];
+  for (const obj of response.Contents ?? []) {
+    if (!obj.Key) continue;
+    const signalId = deriveSignalId(obj.Key, prefix);
+    if (!signalId) continue;
+
+    const summary: SignalSummary = {
+      signalId,
+      signalType: opts.signalType,
+      key: obj.Key,
+      lastModified: obj.LastModified ?? new Date(0),
+      size: obj.Size ?? 0,
+    };
+
+    if (opts.includeFrontmatter) {
+      try {
+        const get = await client.send(
+          new GetObjectCommand({
+            Bucket: opts.entity.bucketName,
+            Key: obj.Key,
+          }),
+        );
+        const text = await streamToString(get.Body);
+        const fm = parseFrontmatter(text);
+        if (fm) {
+          summary.frontmatter = fm;
+          const sourceRef = fm.source_ref;
+          if (typeof sourceRef === "string") {
+            summary.sourceRef = sourceRef;
+          }
+        }
+      } catch {
+        // Frontmatter fetch is best-effort; leave undefined on failure.
+      }
+    }
+
+    entries.push(summary);
+  }
+
+  return {
+    entries,
+    nextToken: response.IsTruncated ? response.NextContinuationToken : undefined,
+  };
+}

package/src/signals/parse.ts

@@ -0,0 +1,8 @@
+/**
+ * YAML frontmatter parsing for signal markdown documents.
+ *
+ * Reuses the same logic as sources/parse.ts — both surfaces emit the
+ * same `---\nYAML\n---\nbody` shape from the pipeline.
+ */
+
+export { parseMarkdown, parseFrontmatter, type ParsedMarkdown } from "../sources/parse.js";

package/src/signals/types.ts

@@ -0,0 +1,74 @@
+/**
+ * Public types for the signals read surface.
+ *
+ * Mirrors the sources types so downstream consumers (CLI, MCP) get a
+ * uniform shape across both surfaces. `sourceRef`, `citations`, and
+ * `entity_refs` are surfaced as typed fields on the full document
+ * because they're the cross-reference primitives an agent needs.
+ */
+
+import type { SignalType } from "../schemas/signal-types.js";
+import type { EntityContext } from "../types.js";
+
+/**
+ * Lightweight summary returned by listSignals(). `frontmatter` and
+ * `sourceRef` are only populated when the caller passes
+ * `includeFrontmatter: true`.
+ */
+export interface SignalSummary {
+  /** Stable ID derived from the object key (filename minus `.md`). */
+  signalId: string;
+  signalType: SignalType;
+  /** Full S3 object key (e.g. "signals/action_item/foo.md"). */
+  key: string;
+  lastModified: Date;
+  size: number;
+  /** Convenience field parsed from frontmatter.source_ref when includeFrontmatter:true. */
+  sourceRef?: string;
+  /** Parsed YAML frontmatter — only present when includeFrontmatter:true. */
+  frontmatter?: Record<string, unknown>;
+}
+
+/**
+ * Full signal document returned by getSignal().
+ *
+ * `sourceRef`, `citations`, and `entity_refs` are surfaced as typed
+ * fields so callers don't have to fish inside `frontmatter`. They're
+ * still present on `frontmatter` too.
+ */
+export interface SignalDocument {
+  key: string;
+  /** Parsed YAML frontmatter, or `null` if the document had no frontmatter block. */
+  frontmatter: Record<string, unknown> | null;
+  /** Markdown body (everything after the closing `---`). */
+  body: string;
+  /** frontmatter.source_ref, if present and a string. */
+  sourceRef?: string;
+  /** frontmatter.citations[], if present and an array. */
+  citations?: string[];
+  /** frontmatter.entity_refs[], if present and an array. */
+  entityRefs?: string[];
+}
+
+export interface ListSignalsOptions {
+  entity: EntityContext;
+  signalType: SignalType;
+  /** Max keys per page; defaults to S3 default (1000). */
+  limit?: number;
+  /** Opaque continuation token returned by a prior page. */
+  continuationToken?: string;
+  /** When true, fetches+parses each entry's frontmatter (extra GETs). */
+  includeFrontmatter?: boolean;
+}
+
+export interface ListSignalsResult {
+  entries: SignalSummary[];
+  /** Present when the result was truncated; pass back as continuationToken. */
+  nextToken?: string;
+}
+
+export interface GetSignalOptions {
+  entity: EntityContext;
+  signalType: SignalType;
+  signalId: string;
+}

package/src/sources/get.test.ts

@@ -0,0 +1,166 @@
+/**
+ * Unit tests for sources/get.ts.
+ */
+
+import { describe, it, expect, beforeEach, afterEach } from "vitest";
+import { GetObjectCommand, type S3Client } from "@aws-sdk/client-s3";
+import { getSource, SourceNotFoundError } from "./get.js";
+import {
+  _setSourcesS3Factory,
+  _resetSourcesS3Factory,
+} from "./internals.js";
+import type { EntityContext } from "../types.js";
+import { InvalidSourceChannelError } from "../schemas/source-channels.js";
+
+// ---------------------------------------------------------------------------
+// Fixtures
+// ---------------------------------------------------------------------------
+
+const ENTITY: EntityContext = {
+  uid: "cmp_test_001",
+  slug: "test",
+  bucketName: "hq-vault-test",
+  region: "us-east-1",
+  credentials: {
+    accessKeyId: "ASIATEST",
+    secretAccessKey: "secret",
+    sessionToken: "session",
+  },
+  expiresAt: new Date(Date.now() + 3600_000).toISOString(),
+};
+
+const MEETING_MD = `---
+source_id: abc
+source_type: meeting
+title: Q1 Planning
+---
+
+Body text line 1.
+Body text line 2.
+`;
+
+const RAW_PAYLOAD = { foo: "bar", n: 42 };
+
+function installStub(objects: Record<string, string>): void {
+  async function send(command: unknown): Promise<unknown> {
+    if (command instanceof GetObjectCommand) {
+      const { Key } = command.input as { Key: string };
+      const content = objects[Key];
+      if (content === undefined) {
+        const err = new Error(`NoSuchKey: ${Key}`);
+        (err as Error & { name: string }).name = "NoSuchKey";
+        throw err;
+      }
+      async function* stream(): AsyncIterable<Uint8Array> {
+        yield Buffer.from(content, "utf-8");
+      }
+      return { Body: stream(), $metadata: {} };
+    }
+    throw new Error(`unhandled command`);
+  }
+  _setSourcesS3Factory(() => ({ send } as unknown as S3Client));
+}
+
+beforeEach(() => _resetSourcesS3Factory());
+afterEach(() => _resetSourcesS3Factory());
+
+// ---------------------------------------------------------------------------
+// Tests
+// ---------------------------------------------------------------------------
+
+describe("getSource", () => {
+  it("happy path: parses frontmatter and body", async () => {
+    installStub({ "sources/meeting/abc.md": MEETING_MD });
+
+    const doc = await getSource({
+      entity: ENTITY,
+      channel: "meeting",
+      sourceId: "abc",
+    });
+
+    expect(doc.key).toBe("sources/meeting/abc.md");
+    expect(doc.frontmatter).toEqual({
+      source_id: "abc",
+      source_type: "meeting",
+      title: "Q1 Planning",
+    });
+    expect(doc.body).toContain("Body text line 1.");
+    expect(doc.body).toContain("Body text line 2.");
+    expect(doc.raw).toBeUndefined();
+  });
+
+  it("throws SourceNotFoundError on NoSuchKey", async () => {
+    installStub({});
+
+    await expect(
+      getSource({ entity: ENTITY, channel: "meeting", sourceId: "missing" }),
+    ).rejects.toBeInstanceOf(SourceNotFoundError);
+
+    await expect(
+      getSource({ entity: ENTITY, channel: "meeting", sourceId: "missing" }),
+    ).rejects.toThrow(/sources\/meeting\/missing\.md/);
+  });
+
+  it("includeRaw:true fetches the .raw.json sibling", async () => {
+    installStub({
+      "sources/meeting/abc.md": MEETING_MD,
+      "sources/meeting/abc.raw.json": JSON.stringify(RAW_PAYLOAD),
+    });
+
+    const doc = await getSource({
+      entity: ENTITY,
+      channel: "meeting",
+      sourceId: "abc",
+      includeRaw: true,
+    });
+
+    expect(doc.raw).toEqual(RAW_PAYLOAD);
+  });
+
+  it("includeRaw:true tolerates missing .raw.json (raw stays undefined)", async () => {
+    installStub({ "sources/meeting/abc.md": MEETING_MD });
+
+    const doc = await getSource({
+      entity: ENTITY,
+      channel: "meeting",
+      sourceId: "abc",
+      includeRaw: true,
+    });
+
+    expect(doc.frontmatter).not.toBeNull();
+    expect(doc.raw).toBeUndefined();
+  });
+
+  it("rejects an invalid channel via assertSourceChannel BEFORE any S3 call", async () => {
+    let called = 0;
+    _setSourcesS3Factory(() => ({
+      send: async () => {
+        called++;
+        return {};
+      },
+    } as unknown as S3Client));
+
+    await expect(
+      getSource({
+        entity: ENTITY,
+        // @ts-expect-error — invalid channel intentional
+        channel: "pigeon",
+        sourceId: "abc",
+      }),
+    ).rejects.toBeInstanceOf(InvalidSourceChannelError);
+    expect(called).toBe(0);
+  });
+
+  it("returns frontmatter:null for malformed (no frontmatter block) documents", async () => {
+    installStub({ "sources/meeting/plain.md": "Just a body, no frontmatter." });
+
+    const doc = await getSource({
+      entity: ENTITY,
+      channel: "meeting",
+      sourceId: "plain",
+    });
+
+    expect(doc.frontmatter).toBeNull();
+    expect(doc.body).toBe("Just a body, no frontmatter.");
+  });
+});

package/src/sources/get.ts

@@ -0,0 +1,75 @@
+/**
+ * getSource — fetch a single source document (and optionally its `.raw.json`
+ * sibling) from an entity bucket and return parsed frontmatter + body.
+ */
+
+import { GetObjectCommand } from "@aws-sdk/client-s3";
+import { assertSourceChannel } from "../schemas/source-channels.js";
+import { getS3Client, streamToString } from "./internals.js";
+import { parseMarkdown } from "./parse.js";
+import type { GetSourceOptions, SourceDocument } from "./types.js";
+
+export class SourceNotFoundError extends Error {
+  constructor(public readonly key: string) {
+    super(`Source not found: ${key}`);
+    this.name = "SourceNotFoundError";
+  }
+}
+
+function isNoSuchKey(err: unknown): boolean {
+  if (!err || typeof err !== "object") return false;
+  const name = (err as { name?: unknown }).name;
+  // AWS SDK v3 throws either a `NoSuchKey` exception or a `NotFound` head error.
+  return name === "NoSuchKey" || name === "NoSuchKeyException";
+}
+
+export async function getSource(opts: GetSourceOptions): Promise<SourceDocument> {
+  // Validate channel BEFORE any S3 call (acceptance criterion).
+  assertSourceChannel(opts.channel);
+
+  const client = getS3Client(opts.entity);
+  const key = `sources/${opts.channel}/${opts.sourceId}.md`;
+
+  let body: string;
+  try {
+    const response = await client.send(
+      new GetObjectCommand({
+        Bucket: opts.entity.bucketName,
+        Key: key,
+      }),
+    );
+    body = await streamToString(response.Body);
+  } catch (err) {
+    if (isNoSuchKey(err)) {
+      throw new SourceNotFoundError(key);
+    }
+    throw err;
+  }
+
+  const parsed = parseMarkdown(body);
+
+  const document: SourceDocument = {
+    key,
+    frontmatter: parsed.frontmatter,
+    body: parsed.body,
+  };
+
+  if (opts.includeRaw) {
+    const rawKey = `sources/${opts.channel}/${opts.sourceId}.raw.json`;
+    try {
+      const rawResponse = await client.send(
+        new GetObjectCommand({
+          Bucket: opts.entity.bucketName,
+          Key: rawKey,
+        }),
+      );
+      const rawText = await streamToString(rawResponse.Body);
+      document.raw = JSON.parse(rawText);
+    } catch (err) {
+      if (!isNoSuchKey(err)) throw err;
+      // Raw sibling is optional; leave undefined when absent.
+    }
+  }
+
+  return document;
+}