@geraldmaron/construct 1.4.2 → 1.5.0

package/lib/providers/feedback/index.mjs

@@ -0,0 +1,392 @@
+/**
+ * lib/providers/feedback/index.mjs — Feedback / customer-input data-source provider.
+ *
+ * Reads a drop-directory of JSONL or CSV files (survey exports, support-ticket
+ * dumps, user-research notes) and normalizes every row into a `feedback-item`
+ * record: { id, text, author, date, source, tags, provenance }. Field mapping
+ * is caller-configured so arbitrary export schemas can map onto the same
+ * shape; only `id` and `text` are required in the mapping, the rest default
+ * to null/empty when unmapped or absent.
+ *
+ * Capabilities: read, search.
+ *
+ * Config (per call):
+ *   - root: absolute or relative path to the drop-directory (required)
+ *   - format: 'jsonl' | 'csv' (required)
+ *   - fields: { id, text, author?, date?, source?, tags? } column/key mapping (required)
+ *
+ * Malformed rows (missing required mapped fields, unparsable JSON/CSV lines)
+ * are skipped, never thrown — each skip increments a per-file counter and a
+ * single warning summarizing the count is emitted per file via console.warn.
+ *
+ * Trust: feedback content originates from external, unauthenticated sources
+ * (customers, survey respondents) with no authorship verification available
+ * to Construct. Every item is stamped EXTERNAL_UNAUTHENTICATED per the N1
+ * trust taxonomy (lib/security/trust.mjs) — highest injection risk, must be
+ * wrapped before entering model context.
+ */
+
+import { existsSync, statSync, readdirSync, readFileSync } from 'node:fs';
+import { resolve, isAbsolute, join, extname } from 'node:path';
+import { TRUST_LEVELS, stampTrust } from '../../security/trust.mjs';
+
+/**
+ * Resolve and validate the configured drop-directory root.
+ */
+function resolveRoot(root) {
+  if (!root) return { ok: false, reason: 'config.root required' };
+  const rootPath = isAbsolute(root) ? root : resolve(process.cwd(), root);
+  if (!existsSync(rootPath)) {
+    return { ok: false, reason: `root does not exist: ${rootPath}` };
+  }
+  const stat = statSync(rootPath);
+  if (!stat.isDirectory()) {
+    return { ok: false, reason: `root is not a directory: ${rootPath}` };
+  }
+  return { ok: true, path: rootPath };
+}
+
+/**
+ * List files under root matching the configured format's extension.
+ */
+function listFormatFiles(rootPath, format) {
+  const ext = format === 'csv' ? '.csv' : '.jsonl';
+  let entries;
+  try {
+    entries = readdirSync(rootPath, { withFileTypes: true });
+  } catch {
+    return [];
+  }
+  return entries
+    .filter((e) => e.isFile() && extname(e.name).toLowerCase() === ext)
+    .map((e) => e.name)
+    .sort();
+}
+
+/**
+ * Minimal RFC-4180-ish CSV line splitter supporting quoted fields, escaped
+ * quotes (""), and commas/newlines within quotes. No external dependency —
+ * the repo has none for CSV, and drop-dir exports are expected to be simple.
+ */
+function parseCsv(content) {
+  const rows = [];
+  let row = [];
+  let field = '';
+  let inQuotes = false;
+  let i = 0;
+
+  while (i < content.length) {
+    const ch = content[i];
+
+    if (inQuotes) {
+      if (ch === '"') {
+        if (content[i + 1] === '"') {
+          field += '"';
+          i += 2;
+          continue;
+        }
+        inQuotes = false;
+        i += 1;
+        continue;
+      }
+      field += ch;
+      i += 1;
+      continue;
+    }
+
+    if (ch === '"') {
+      inQuotes = true;
+      i += 1;
+      continue;
+    }
+    if (ch === ',') {
+      row.push(field);
+      field = '';
+      i += 1;
+      continue;
+    }
+    if (ch === '\r') {
+      i += 1;
+      continue;
+    }
+    if (ch === '\n') {
+      row.push(field);
+      rows.push(row);
+      row = [];
+      field = '';
+      i += 1;
+      continue;
+    }
+    field += ch;
+    i += 1;
+  }
+
+  if (field.length > 0 || row.length > 0) {
+    row.push(field);
+    rows.push(row);
+  }
+
+  return rows;
+}
+
+/**
+ * Extract a mapped field from a source object; returns undefined when the
+ * mapping key is unset or the source object lacks that key.
+ */
+function mapField(sourceObj, mappingKey) {
+  if (!mappingKey) return undefined;
+  return sourceObj[mappingKey];
+}
+
+/**
+ * Normalize a raw mapped row into the canonical feedback-item shape, minus
+ * provenance and trust (added by the caller). Returns null when required
+ * fields (id, text) are missing — caller counts this as a skip.
+ */
+function normalizeRow(rawObj, fields) {
+  const id = mapField(rawObj, fields.id);
+  const text = mapField(rawObj, fields.text);
+
+  if (id === undefined || id === null || String(id).trim() === '') return null;
+  if (text === undefined || text === null || String(text).trim() === '') return null;
+
+  const author = mapField(rawObj, fields.author);
+  const date = mapField(rawObj, fields.date);
+  const source = mapField(rawObj, fields.source);
+  const rawTags = fields.tags ? mapField(rawObj, fields.tags) : undefined;
+
+  let tags = [];
+  if (Array.isArray(rawTags)) {
+    tags = rawTags.map(String);
+  } else if (typeof rawTags === 'string' && rawTags.trim() !== '') {
+    tags = rawTags.split(/[;,|]/).map((t) => t.trim()).filter(Boolean);
+  }
+
+  return {
+    id: String(id),
+    text: String(text),
+    author: author != null ? String(author) : null,
+    date: date != null ? String(date) : null,
+    source: source != null ? String(source) : null,
+    tags,
+  };
+}
+
+/**
+ * Parse a single JSONL file into feedback-items. Malformed lines (unparsable
+ * JSON, missing required fields) are skipped; the count is returned so the
+ * caller can emit one warning per file.
+ */
+function readJsonlFile(filePath, fields) {
+  const items = [];
+  let skipped = 0;
+  const content = readFileSync(filePath, 'utf8');
+  const lines = content.split(/\r?\n/);
+
+  for (let lineNo = 0; lineNo < lines.length; lineNo++) {
+    const line = lines[lineNo].trim();
+    if (line === '') continue;
+
+    let parsed;
+    try {
+      parsed = JSON.parse(line);
+    } catch {
+      skipped += 1;
+      continue;
+    }
+
+    if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) {
+      skipped += 1;
+      continue;
+    }
+
+    const normalized = normalizeRow(parsed, fields);
+    if (!normalized) {
+      skipped += 1;
+      continue;
+    }
+
+    items.push({ ...normalized, row: lineNo + 1 });
+  }
+
+  return { items, skipped };
+}
+
+/**
+ * Parse a single CSV file into feedback-items. First row is the header.
+ * Malformed rows (column-count mismatch, missing required fields) are
+ * skipped; the count is returned so the caller can emit one warning per file.
+ */
+function readCsvFile(filePath, fields) {
+  const items = [];
+  let skipped = 0;
+  const content = readFileSync(filePath, 'utf8');
+  const rows = parseCsv(content);
+
+  if (rows.length === 0) return { items, skipped };
+
+  const header = rows[0];
+
+  for (let rowNo = 1; rowNo < rows.length; rowNo++) {
+    const row = rows[rowNo];
+    if (row.length === 1 && row[0] === '') continue;
+
+    if (row.length !== header.length) {
+      skipped += 1;
+      continue;
+    }
+
+    const rawObj = {};
+    for (let c = 0; c < header.length; c++) {
+      rawObj[header[c]] = row[c];
+    }
+
+    const normalized = normalizeRow(rawObj, fields);
+    if (!normalized) {
+      skipped += 1;
+      continue;
+    }
+
+    items.push({ ...normalized, row: rowNo + 1 });
+  }
+
+  return { items, skipped };
+}
+
+/**
+ * Validate the required config shape shared by read/search.
+ */
+function validateConfig(config) {
+  const root = config?.root;
+  const format = config?.format;
+  const fields = config?.fields;
+
+  const resolved = resolveRoot(root);
+  if (!resolved.ok) throw new Error(`feedback: ${resolved.reason}`);
+
+  if (format !== 'jsonl' && format !== 'csv') {
+    throw new Error(`feedback: config.format must be 'jsonl' or 'csv' (got '${format}')`);
+  }
+
+  if (!fields || typeof fields !== 'object' || !fields.id || !fields.text) {
+    throw new Error('feedback: config.fields.id and config.fields.text are required');
+  }
+
+  return { rootPath: resolved.path, format, fields };
+}
+
+/**
+ * Read every file of the configured format under root, normalize rows to
+ * feedback-items, stamp provenance + N1 trust, and warn once per file with
+ * any skipped-row count.
+ */
+async function readAllFeedback(config) {
+  const { rootPath, format, fields } = validateConfig(config);
+
+  const files = listFormatFiles(rootPath, format);
+  const results = [];
+
+  for (const fileName of files) {
+    const filePath = join(rootPath, fileName);
+    const { items, skipped } = format === 'csv'
+      ? readCsvFile(filePath, fields)
+      : readJsonlFile(filePath, fields);
+
+    if (skipped > 0) {
+      console.warn(`feedback provider: skipped ${skipped} malformed row(s) in ${fileName}`);
+    }
+
+    for (const item of items) {
+      const { row, ...rest } = item;
+      results.push(stampTrust(
+        {
+          ...rest,
+          provenance: { file: fileName, row },
+        },
+        TRUST_LEVELS.EXTERNAL_UNAUTHENTICATED,
+        `feedback:${fileName}`,
+      ));
+    }
+  }
+
+  return results;
+}
+
+/**
+ * Search feedback-item text for a substring match (case-insensitive).
+ */
+async function searchFeedback(config, query) {
+  if (!query || typeof query !== 'string') {
+    throw new Error('feedback.search: query required (substring)');
+  }
+
+  const items = await readAllFeedback(config);
+  const needle = query.toLowerCase();
+  return items.filter((item) => item.text.toLowerCase().includes(needle));
+}
+
+export function create({ env = process.env } = {}) {
+  return {
+    meta: {
+      id: 'feedback',
+      displayName: 'Feedback / Customer Input',
+      capabilities: ['read', 'search'],
+      description: 'Read and search customer feedback exports (JSONL/CSV) from a drop-directory.',
+    },
+
+    configSchema: {
+      $schema: 'https://json-schema.org/draft/2020-12/schema',
+      type: 'object',
+      properties: {
+        root: {
+          type: 'string',
+          description: 'Path to the feedback drop-directory (absolute or relative to cwd)',
+        },
+        format: {
+          type: 'string',
+          enum: ['jsonl', 'csv'],
+          description: 'Format of files to read from the drop-directory',
+        },
+        fields: {
+          type: 'object',
+          description: 'Field mapping from source column/key names to feedback-item shape',
+          properties: {
+            id: { type: 'string', description: 'Source key/column mapped to feedback-item id' },
+            text: { type: 'string', description: 'Source key/column mapped to feedback-item text' },
+            author: { type: 'string', description: 'Source key/column mapped to feedback-item author' },
+            date: { type: 'string', description: 'Source key/column mapped to feedback-item date' },
+            source: { type: 'string', description: 'Source key/column mapped to feedback-item source' },
+            tags: { type: 'string', description: 'Source key/column mapped to feedback-item tags' },
+          },
+          required: ['id', 'text'],
+        },
+      },
+      required: ['root', 'format', 'fields'],
+    },
+
+    async health(config) {
+      const root = config?.root;
+      if (!root) {
+        return { ok: false, detail: 'config.root not set' };
+      }
+
+      const resolved = resolveRoot(root);
+      if (!resolved.ok) {
+        return { ok: false, detail: resolved.reason };
+      }
+
+      try {
+        const format = config?.format === 'csv' ? 'csv' : 'jsonl';
+        const files = listFormatFiles(resolved.path, format);
+        return { ok: true, detail: `drop-directory readable (${files.length} ${format} file(s))` };
+      } catch (err) {
+        return { ok: false, detail: err.message };
+      }
+    },
+
+    read: readAllFeedback,
+    search: searchFeedback,
+  };
+}
+
+export default create;

package/lib/providers/filter-audit.mjs

@@ -0,0 +1,68 @@
+/**
+ * lib/providers/filter-audit.mjs — per-poll provider filter audit log.
+ *
+ * ADR-0060 requires every poll to record, per source, the filter that was
+ * enforced and how many items it dropped. Appends one JSONL line per source
+ * per poll to `<rootDir>/.cx/providers/filter-audit.jsonl` with the
+ * acceptance-criteria field names `{provider, instance, filterHash, matched,
+ * dropped}`, plus the ADR's own `fetched`/`admitted` and `pushdown` fields
+ * for the fuller audit shape. Rotation uses `appendWithRotationSync`
+ * directly (not the shared `appendBounded` channel registry) to keep this
+ * change scoped to `lib/providers/*`.
+ */
+import { existsSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
+
+import { appendWithRotationSync } from '../logging/rotate.mjs';
+
+const AUDIT_SUBDIR = ['.cx', 'providers'];
+const AUDIT_FILE = 'filter-audit.jsonl';
+
+// 10 MB per shard, 5 shards kept — a filter-audit line is written once per
+// source per poll, so this comfortably covers years of daemon uptime.
+
+const MAX_BYTES = 10 * 1024 * 1024;
+const MAX_SEGMENTS = 5;
+
+export function filterAuditPath(rootDir) {
+  return join(rootDir, ...AUDIT_SUBDIR, AUDIT_FILE);
+}
+
+/**
+ * Append one filter-audit record. `matched`/`dropped` are the
+ * acceptance-criteria field names; `fetched`/`admitted` mirror the ADR-0060
+ * audit shape (`admitted` === `matched`) so both vocabularies are queryable
+ * from the same line without a reader needing to know which one shipped.
+ */
+export function recordFilterAudit(rootDir, {
+  provider,
+  instance = null,
+  filterHash: hash = null,
+  scope = null,
+  predicates = null,
+  nativeQuery = null,
+  pushdown = false,
+  fetched,
+  matched,
+}) {
+  const record = {
+    ts: new Date().toISOString(),
+    provider,
+    instance,
+    filterHash: hash,
+    filterApplied: { scope, predicates, nativeQuery, pushdown, fetched, admitted: matched },
+    matched,
+    dropped: fetched - matched,
+  };
+  appendWithRotationSync(filterAuditPath(rootDir), JSON.stringify(record) + '\n', {
+    maxBytes: MAX_BYTES,
+    maxSegments: MAX_SEGMENTS,
+  });
+  return record;
+}
+
+export function readFilterAudit(rootDir) {
+  const path = filterAuditPath(rootDir);
+  if (!existsSync(path)) return [];
+  return readFileSync(path, 'utf8').split('\n').filter(Boolean).map((line) => JSON.parse(line));
+}

package/lib/providers/filter-schema.mjs

@@ -0,0 +1,54 @@
+/**
+ * lib/providers/filter-schema.mjs — provider filter block JSON schema (ADR-0060).
+ *
+ * Ships the normalized filter grammar every provider kind may accept:
+ * `scope` (per-kind container allowlist), `predicates` (portable field-level
+ * filter core), and `nativeQuery` (provider-native passthrough escape hatch).
+ * Per-kind manifest `configSchema` fields reference this via `$ref`; per-kind
+ * `SCOPE_FIELDS_BY_PROVIDER` in `lib/providers/contract.mjs` declares which
+ * `scope` keys a given provider kind actually honors.
+ *
+ * Data-only module (schema + enums), consumed by `validateFilterConfig()`
+ * in `lib/providers/contract.mjs`. No JSON-Schema engine runs here — the
+ * shape below is the source of truth the hand-rolled validator checks
+ * against, keeping schema and validator from drifting silently apart.
+ */
+
+export const FILTER_SCHEMA = Object.freeze({
+  $id: 'construct:provider-filter',
+  type: 'object',
+  additionalProperties: false,
+  properties: {
+    scope: {
+      type: 'object',
+      additionalProperties: false,
+      properties: {
+        projects: { type: 'array', items: { type: 'string' } },
+        repos: { type: 'array', items: { type: 'string' } },
+        repoGlob: { type: 'array', items: { type: 'string' } },
+        channels: { type: 'array', items: { type: 'string' } },
+        spaces: { type: 'array', items: { type: 'string' } },
+      },
+    },
+    predicates: {
+      type: 'object',
+      additionalProperties: false,
+      properties: {
+        assignee: { type: 'array', items: { type: 'string' } },
+        statusCategory: { type: 'array', items: { enum: ['to-do', 'in-progress', 'done'] } },
+        priority: { type: 'array', items: { type: 'string' } },
+        label: { type: 'array', items: { type: 'string' } },
+        updatedSince: { type: 'string' },
+      },
+    },
+    nativeQuery: { type: 'string' },
+  },
+});
+
+export const SCOPE_KEYS = Object.freeze(['projects', 'repos', 'repoGlob', 'channels', 'spaces']);
+
+export const PREDICATE_KEYS = Object.freeze(['assignee', 'statusCategory', 'priority', 'label', 'updatedSince']);
+
+export const STATUS_CATEGORIES = Object.freeze(['to-do', 'in-progress', 'done']);
+
+export const TOP_LEVEL_KEYS = Object.freeze(['scope', 'predicates', 'nativeQuery']);

package/lib/providers/github/index.mjs

@@ -14,10 +14,21 @@
  * The provider is intentionally narrow in v1 — issue + PR + code search +
  * a single repo metadata read. Webhook signature verification is in place
  * so consumers can mount a receiver without re-implementing it.
+ *
+ * Scope enforcement: when `repoAllowlist` or `repoAllowGlob` is present in
+ * config, `read()` validates `config.repo` and `search()` validates every
+ * `repo:` qualifier in the query through `validateAllowlist()`
+ * (lib/providers/contract.mjs) before any network call, throwing a typed
+ * `OUT_OF_SCOPE` error on a blocked target. A search query with no `repo:`
+ * qualifier is rejected outright when an allowlist is configured — an
+ * unscoped query could otherwise span the whole credential's visible
+ * surface and defeat the allowlist.
  */
 
 import crypto from 'node:crypto';
 
+import { validateAllowlist } from '../contract.mjs';
+
 const API = 'https://api.github.com';
 
 function authHeader(env) {
@@ -25,6 +36,10 @@ function authHeader(env) {
   return token ? { Authorization: `Bearer ${token}` } : {};
 }
 
+function outOfScopeError(reason) {
+  return Object.assign(new Error(reason), { code: 'OUT_OF_SCOPE' });
+}
+
 async function ghFetch(pathOrUrl, env, init = {}) {
   const url = pathOrUrl.startsWith('http') ? pathOrUrl : `${API}${pathOrUrl}`;
   const headers = {
@@ -83,6 +98,8 @@ export function create({ env = process.env } = {}) {
 
     async read(config) {
       if (!config?.repo) throw new Error('github.read: config.repo required (owner/name)');
+      const check = validateAllowlist('github', config.repo, config);
+      if (!check.allowed) throw outOfScopeError(check.reason);
       return ghFetch(`/repos/${config.repo}`, env);
     },
 
@@ -90,6 +107,20 @@ export function create({ env = process.env } = {}) {
       const kind = config?.kind || 'issues';
       const query = config?.query;
       if (!query) throw new Error('github.search: config.query required');
+
+      const hasAllowlistConfig = (Array.isArray(config?.repoAllowlist) && config.repoAllowlist.length > 0)
+        || (typeof config?.repoAllowGlob === 'string' && config.repoAllowGlob.length > 0);
+      const repoQualifiers = [...query.matchAll(/\brepo:(\S+)/g)].map((m) => m[1]);
+
+      if (hasAllowlistConfig && repoQualifiers.length === 0) {
+        throw outOfScopeError('github.search: repoAllowlist/repoAllowGlob is configured; query must include a repo: qualifier to enforce scope');
+      }
+      for (const repo of repoQualifiers) {
+        const target = repo.includes('/') ? repo.split('/').pop() : repo;
+        const check = validateAllowlist('github', target, config);
+        if (!check.allowed) throw outOfScopeError(check.reason);
+      }
+
       const path = kind === 'code'
         ? `/search/code?q=${encodeURIComponent(query)}`
         : kind === 'prs'