@walkeros/config 4.7.0-next-1790245733116 → 4.7.0

package/CHANGELOG.md

@@ -1,6 +1,28 @@
 # @walkeros/config
 
-## 4.7.0-next-1790245733116
+## 4.7.0
+
+### Minor Changes
+
+- 64b06de: Packages with several exports, such as the GCP and AWS sources and
+  destinations, now publish a settings schema per export. `walkeros validate`
+  and MCP `package_get` use the schema of the export a step imports. Pub/Sub,
+  BigQuery, SNS and SQS steps pinned to this release or later are checked
+  instead of skipped; older pins still report a skip.
+
+### Patch Changes
+
+- 74821ed: Simulate records every vendor call through one recorder, `observeEnv`
+  from `@walkeros/core`, and runs offline: BigQuery, Data Manager and the
+  Sheets, GCS and S3 stores reach the network through `env`. Client and `Env`
+  types of BigQuery, Firehose, PostHog, SQS and Pub/Sub are structural.
+  `@walkeros/cli` exports `resolveExportName` and `selectDevExamples` and drops
+  `findExample` and `compareOutput`.
+- 74821ed: Simulate starts only the simulated step, and a simulated transformer
+  continues through its `next`. Source simulations list the commands a source
+  issues as `elb` calls; `--page-url` sets the web page. Express flows need no
+  port. Packages with several exports use per-export examples and mocks, and a
+  destination without a mock env is refused instead of calling the vendor.
 
 ## 4.6.1
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@walkeros/config",
-  "version": "4.7.0-next-1790245733116",
+  "version": "4.7.0",
   "type": "module",
   "description": "Shared development configuration for walkerOS packages (TypeScript, ESLint, Jest, tsup)",
   "license": "MIT",

package/tsup/__tests__/credentialGuard.test.mjs

@@ -0,0 +1,136 @@
+import {
+  assertNoPublishedCredentials,
+  findPublishedCredential,
+} from '../credentialGuard.mjs';
+
+const keyMaterial =
+  'MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC7kJ8sL2mNqRtVxYz'.repeat(
+    3,
+  );
+const wrapped = keyMaterial.match(/.{1,64}/g).join('\n');
+const realPem = `-----BEGIN PRIVATE KEY-----\n${wrapped}\n-----END PRIVATE KEY-----\n`;
+
+describe('assertNoPublishedCredentials', () => {
+  it.each([
+    ['a PEM with real key material', realPem, 'PEM private key'],
+    [
+      'a PEM with escaped newlines',
+      realPem.replace(/\n/g, '\\n'),
+      'PEM private key',
+    ],
+    ['a live Stripe key', 'sk_live_51Habcdefghijkl', 'live key token sk_live_'],
+    ['a GitHub token', 'token ghp_abcdefghijklmnop', 'live key token ghp_'],
+    ['a Slack bot token', 'xoxb-1234-5678-abcd', 'live key token xoxb-'],
+    ['an AWS access key id', 'AKIAABCDEFGHIJKLMNOP', 'live key token AKIA'],
+  ])(
+    'throws on %s with the path and fix, never the value',
+    (_l, value, rule) => {
+      const examples = { step: { in: { settings: { credentials: value } } } };
+      let message = '';
+      try {
+        assertNoPublishedCredentials('@walkeros/x', { examples });
+      } catch (error) {
+        message = error.message;
+      }
+      expect(message).toBe(
+        `@walkeros/x: examples.step.in.settings.credentials looks like a real credential (${rule}). Replace it with a placeholder, e.g. sk_test_..., or a PEM with fewer than 64 characters of key material.`,
+      );
+      expect(message).not.toContain(value);
+    },
+  );
+
+  it.each(['github_pat_', 'ghu_', 'ghs_', 'ghr_'])(
+    'throws on a GitHub %s token',
+    (prefix) => {
+      const value = `${prefix}${'a1B2c3D4e5'.repeat(3)}f6G7h8`;
+      expect(() =>
+        assertNoPublishedCredentials('@walkeros/x', {
+          examples: { step: { in: value } },
+        }),
+      ).toThrow(`(live key token ${prefix})`);
+    },
+  );
+
+  it.each([
+    [
+      'a credential as an object key',
+      { step: { in: { sk_live_51Habcdefghijkl: 'placeholder' } } },
+      'examples.step.in.<key>',
+      'live key token sk_live_',
+      ['sk_live_51Habcdefghijkl'],
+    ],
+    [
+      'a credential key above a credential value',
+      {
+        step: {
+          in: {
+            ghs_abcdefghijklmnopqrstuvwxyz0123456789: {
+              token: 'github_pat_abcdefghijklmnopqrstuvwxyz0123456789',
+            },
+          },
+        },
+      },
+      'examples.step.in.<key>',
+      'live key token ghs_',
+      [
+        'ghs_abcdefghijklmnopqrstuvwxyz0123456789',
+        'github_pat_abcdefghijklmnopqrstuvwxyz0123456789',
+      ],
+    ],
+  ])(
+    'throws on %s without the key or value text',
+    (_l, examples, path, rule, secrets) => {
+      let message = '';
+      try {
+        assertNoPublishedCredentials('@walkeros/x', { examples });
+      } catch (error) {
+        message = error.message;
+      }
+      expect(message).toBe(
+        `@walkeros/x: ${path} looks like a real credential (${rule}). Replace it with a placeholder, e.g. sk_test_..., or a PEM with fewer than 64 characters of key material.`,
+      );
+      for (const secret of secrets) expect(message).not.toContain(secret);
+    },
+  );
+
+  it('passes a plain header key with a placeholder value', () => {
+    expect(() =>
+      assertNoPublishedCredentials('@walkeros/x', {
+        examples: {
+          step: { in: { headers: { Authorization: 'Bearer ***' } } },
+        },
+      }),
+    ).not.toThrow();
+  });
+
+  it('names the section and array index of the hit', () => {
+    expect(
+      findPublishedCredential({
+        examples: {},
+        hints: { auth: { examples: ['ok', 'sk_live_abc'] } },
+      }),
+    ).toEqual({
+      path: 'hints.auth.examples.1',
+      rule: 'live key token sk_live_',
+    });
+  });
+
+  it.each([
+    ['a short PEM placeholder', '-----BEGIN PRIVATE KEY-----\ntest'],
+    [
+      'an escaped short PEM placeholder',
+      '-----BEGIN PRIVATE KEY-----\\ntest\\n-----END PRIVATE KEY-----\\n',
+    ],
+    ['a test Stripe key', 'sk_test_x'],
+    ['the AWS documentation key id', 'AKIDEXAMPLE'],
+    ['a prefix inside a word', 'mask_live_ and task_ghp_x'],
+  ])('passes %s', (_l, value) => {
+    expect(() =>
+      assertNoPublishedCredentials('@walkeros/x', {
+        examples: { step: { in: value } },
+        exportExamples: { other: { step: { out: [value] } } },
+        hints: undefined,
+      }),
+    ).not.toThrow();
+  });
+});

package/tsup/__tests__/exportMaps.test.mjs

@@ -0,0 +1,129 @@
+import {
+  assertExportMaps,
+  assertStepSettings,
+  publishedExports,
+} from '../exportMaps.mjs';
+
+const two = { destinationA: 'A', destinationB: 'B' };
+const examples = { destinationA: {}, destinationB: {} };
+const schemas = {
+  destinationA: { settings: {} },
+  destinationB: { settings: {}, setup: {} },
+};
+
+describe('assertExportMaps', () => {
+  it.each([
+    ['no declared exports and no maps', undefined, {}],
+    ['a single export and no maps', { destinationA: 'A' }, {}],
+    [
+      'two exports with matching maps',
+      two,
+      { exportExamples: examples, exportSchemas: schemas },
+    ],
+  ])('passes for %s', (_label, declared, maps) => {
+    expect(() => assertExportMaps('@walkeros/x', declared, maps)).not.toThrow();
+  });
+
+  it.each([
+    [
+      'exportSchemas missing',
+      two,
+      { exportExamples: examples },
+      '@walkeros/x: exportSchemas is missing',
+    ],
+    [
+      'exportExamples missing',
+      two,
+      { exportSchemas: schemas },
+      '@walkeros/x: exportExamples is missing',
+    ],
+    [
+      'an export missing from exportSchemas',
+      two,
+      {
+        exportExamples: examples,
+        exportSchemas: { destinationA: { settings: {} } },
+      },
+      '@walkeros/x: exportSchemas is missing export "destinationB"',
+    ],
+    [
+      'an undeclared export in exportSchemas',
+      two,
+      {
+        exportExamples: examples,
+        exportSchemas: { ...schemas, destinationC: { settings: {} } },
+      },
+      '@walkeros/x: exportSchemas has undeclared export "destinationC"',
+    ],
+    [
+      'an exportSchemas entry without settings',
+      two,
+      {
+        exportExamples: examples,
+        exportSchemas: { ...schemas, destinationB: { setup: {} } },
+      },
+      '@walkeros/x: exportSchemas.destinationB has no settings schema',
+    ],
+    [
+      'an export missing from exportExamples',
+      two,
+      { exportExamples: { destinationA: {} }, exportSchemas: schemas },
+      '@walkeros/x: exportExamples is missing export "destinationB"',
+    ],
+    [
+      'maps on a single-export package',
+      { destinationA: 'A' },
+      { exportSchemas: { destinationA: { settings: {} } } },
+      '@walkeros/x: exportSchemas is set but the package declares fewer than two exports',
+    ],
+  ])('throws for %s', (_label, declared, maps, message) => {
+    expect(() => assertExportMaps('@walkeros/x', declared, maps)).toThrow(
+      new Error(message),
+    );
+  });
+});
+
+describe('assertStepSettings', () => {
+  it.each(['source', 'transformer', 'destination', 'store'])(
+    'throws for a %s package without schemas.settings',
+    (type) => {
+      expect(() => assertStepSettings('@walkeros/x', type, {})).toThrow(
+        new Error('@walkeros/x: schemas.settings is missing'),
+      );
+    },
+  );
+
+  it.each([
+    ['a destination with settings', 'destination', { settings: {} }],
+    ['a package without a type', undefined, {}],
+    ['a non-step package', 'core', {}],
+  ])('passes for %s', (_label, type, published) => {
+    expect(() =>
+      assertStepSettings('@walkeros/x', type, published),
+    ).not.toThrow();
+  });
+});
+
+describe('publishedExports', () => {
+  it.each([
+    [
+      'a multi-export package keeps its declared exports',
+      { type: 'destination', exports: two },
+      two,
+    ],
+    [
+      'a single-export package keeps its one declared export',
+      { type: 'destination', exports: { destinationA: 'A' } },
+      { destinationA: 'A' },
+    ],
+    [
+      'a step package without declared exports publishes an empty list',
+      { type: 'source' },
+      {},
+    ],
+    ['a non-step package publishes no list', { type: 'core' }, undefined],
+    ['a package without a type publishes no list', {}, undefined],
+  ])('%s', (_label, walkerOS, expected) => {
+    expect(publishedExports(walkerOS)).toEqual(expected);
+  });
+});

package/tsup/credentialGuard.mjs

@@ -0,0 +1,87 @@
+// Guard for published step metadata: dist/walkerOS.json ships to npm and the
+// hub, so a real credential pasted into an example or a hint must fail the
+// build. Placeholders pass: a PEM with under 64 characters of key material,
+// `sk_test_...`, `AKIDEXAMPLE`.
+
+// Base64 characters of key material a PEM body needs to count as real.
+const MIN_PEM_KEY_CHARS = 64;
+
+// The marker, then the body: base64 characters, whitespace or an escaped
+// `\n`. The alternatives start on different characters, so the scan is linear.
+const RE_PEM_BODY =
+  /-----BEGIN [A-Z ]*PRIVATE KEY-----((?:[A-Za-z0-9+/=]|\s|\\n)*)/g;
+
+const RE_LIVE_TOKEN =
+  /(?:^|[^A-Za-z0-9_])(sk_live_|github_pat_|ghp_|gho_|ghu_|ghs_|ghr_|xoxb-|xoxp-|AKIA[A-Z0-9]{16})/;
+
+// Path segment reported for an object key that breaks a rule: the key text is
+// the credential, so it never appears in the message.
+const KEY_PLACEHOLDER = '<key>';
+
+function countKeyChars(body) {
+  return body.replace(/\\n|\s/g, '').length;
+}
+
+/** The rule a string breaks, or undefined when it looks like a placeholder. */
+function credentialRule(value) {
+  RE_PEM_BODY.lastIndex = 0;
+  let match;
+  while ((match = RE_PEM_BODY.exec(value)) !== null) {
+    if (countKeyChars(match[1]) >= MIN_PEM_KEY_CHARS) return 'PEM private key';
+  }
+  const token = RE_LIVE_TOKEN.exec(value);
+  if (token) {
+    const prefix = token[1].startsWith('AKIA') ? 'AKIA' : token[1];
+    return `live key token ${prefix}`;
+  }
+  return undefined;
+}
+
+function findInValue(value, path) {
+  if (typeof value === 'string') {
+    const rule = credentialRule(value);
+    return rule ? { path, rule } : undefined;
+  }
+  if (Array.isArray(value)) {
+    for (let i = 0; i < value.length; i++) {
+      const hit = findInValue(value[i], `${path}.${i}`);
+      if (hit) return hit;
+    }
+    return undefined;
+  }
+  if (value && typeof value === 'object') {
+    for (const [key, child] of Object.entries(value)) {
+      // A key is published text too: check it before descending.
+      const keyRule = credentialRule(key);
+      if (keyRule) return { path: `${path}.${KEY_PLACEHOLDER}`, rule: keyRule };
+      const hit = findInValue(child, `${path}.${key}`);
+      if (hit) return hit;
+    }
+  }
+  return undefined;
+}
+
+/**
+ * The first serialized string in `sections` (e.g. `{ examples, hints }`) that
+ * looks like a real credential, as `{ path, rule }`; undefined when none does.
+ * The value itself is never returned.
+ */
+function findPublishedCredential(sections) {
+  for (const [name, value] of Object.entries(sections)) {
+    if (value === undefined) continue;
+    const hit = findInValue(value, name);
+    if (hit) return hit;
+  }
+  return undefined;
+}
+
+/** Throw when a published section carries a real-looking credential. */
+function assertNoPublishedCredentials(packageName, sections) {
+  const hit = findPublishedCredential(sections);
+  if (!hit) return;
+  throw new Error(
+    `${packageName}: ${hit.path} looks like a real credential (${hit.rule}). Replace it with a placeholder, e.g. sk_test_..., or a PEM with fewer than 64 characters of key material.`,
+  );
+}
+
+export { findPublishedCredential, assertNoPublishedCredentials };

package/tsup/exportMaps.mjs

@@ -0,0 +1,78 @@
+// Guards for published step metadata in dist/walkerOS.json. A package that
+// declares two or more exports in package.json `walkerOS.exports` ships
+// `exportExamples` and `exportSchemas` keyed by exactly those export names,
+// the default export included, so a reader never falls back to another
+// export's examples or settings schema.
+
+const STEP_TYPES = ['source', 'transformer', 'destination', 'store'];
+
+function isRecord(value) {
+  return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+
+function assertKeys(packageName, mapName, map, declared) {
+  if (!isRecord(map)) throw new Error(`${packageName}: ${mapName} is missing`);
+  for (const name of declared) {
+    if (!(name in map))
+      throw new Error(`${packageName}: ${mapName} is missing export "${name}"`);
+  }
+  for (const name of Object.keys(map)) {
+    if (!declared.includes(name))
+      throw new Error(
+        `${packageName}: ${mapName} has undeclared export "${name}"`,
+      );
+  }
+}
+
+/**
+ * Throw when the per-export maps do not match the declared exports: with two
+ * or more exports both maps list every export and each `exportSchemas` entry
+ * carries a `settings` schema; with fewer, neither map is present.
+ */
+function assertExportMaps(packageName, declaredExports, maps) {
+  const declared = isRecord(declaredExports)
+    ? Object.keys(declaredExports)
+    : [];
+  const { exportExamples, exportSchemas } = maps;
+
+  if (declared.length < 2) {
+    for (const [mapName, map] of [
+      ['exportExamples', exportExamples],
+      ['exportSchemas', exportSchemas],
+    ]) {
+      if (map !== undefined)
+        throw new Error(
+          `${packageName}: ${mapName} is set but the package declares fewer than two exports`,
+        );
+    }
+    return;
+  }
+
+  assertKeys(packageName, 'exportExamples', exportExamples, declared);
+  assertKeys(packageName, 'exportSchemas', exportSchemas, declared);
+  for (const [name, entry] of Object.entries(exportSchemas)) {
+    if (!isRecord(entry) || !isRecord(entry.settings))
+      throw new Error(
+        `${packageName}: exportSchemas.${name} has no settings schema`,
+      );
+  }
+}
+
+/** Throw when a step package (source, transformer, destination, store) publishes no settings schema. */
+function assertStepSettings(packageName, type, schemas) {
+  if (!STEP_TYPES.includes(type)) return;
+  if (!isRecord(schemas) || !isRecord(schemas.settings))
+    throw new Error(`${packageName}: schemas.settings is missing`);
+}
+
+/**
+ * The `$meta.exports` a package publishes: its declared `walkerOS.exports`,
+ * else an empty list for a step package, so a reader can tell a package
+ * without named exports from an older package that published no list.
+ */
+function publishedExports(walkerOS) {
+  if (isRecord(walkerOS.exports)) return walkerOS.exports;
+  return STEP_TYPES.includes(walkerOS.type) ? {} : undefined;
+}
+
+export { assertExportMaps, assertStepSettings, publishedExports };

package/tsup/index.mjs

@@ -2,6 +2,12 @@ import { defineConfig } from 'tsup';
 import { readFileSync, writeFileSync, mkdirSync, existsSync } from 'fs';
 import { resolve, join } from 'path';
 import { pathToFileURL } from 'url';
+import { assertNoPublishedCredentials } from './credentialGuard.mjs';
+import {
+  assertExportMaps,
+  assertStepSettings,
+  publishedExports,
+} from './exportMaps.mjs';
 
 const baseConfig = {
   entry: ['src/index.ts'],
@@ -16,7 +22,10 @@ const getVersion = () => {
     const pkg = JSON.parse(readFileSync(packagePath, 'utf8'));
     return pkg.version || '0.0.0';
   } catch (error) {
-    console.warn('Could not read package.json for version injection:', error.message);
+    console.warn(
+      'Could not read package.json for version injection:',
+      error.message,
+    );
     return '0.0.0';
   }
 };
@@ -140,7 +149,9 @@ const buildDev = (customConfig = {}) => {
       // Check the built module exists
       const devMjsPath = resolve(cwd, 'dist/dev.mjs');
       if (!existsSync(devMjsPath)) {
-        console.warn('[buildDev] dist/dev.mjs not found, skipping walkerOS.json generation');
+        console.warn(
+          '[buildDev] dist/dev.mjs not found, skipping walkerOS.json generation',
+        );
         return;
       }
 
@@ -173,7 +184,8 @@ const buildDev = (customConfig = {}) => {
           : [walkerOS.platform];
       }
       if (walkerOS.renderer) meta.renderer = walkerOS.renderer;
-      if (walkerOS.exports) meta.exports = walkerOS.exports;
+      const exports = publishedExports(walkerOS);
+      if (exports) meta.exports = exports;
 
       // Docs URL (manual, from walkerOS.docs in package.json)
       if (walkerOS.docs) meta.docs = walkerOS.docs;
@@ -181,7 +193,9 @@ const buildDev = (customConfig = {}) => {
       // Source URL (auto-derived from repository.directory)
       const repo = pkg.repository;
       if (repo?.directory) {
-        const repoUrl = (repo.url || '').replace(/^git\+/, '').replace(/\.git$/, '');
+        const repoUrl = (repo.url || '')
+          .replace(/^git\+/, '')
+          .replace(/\.git$/, '');
         if (repoUrl) meta.source = `${repoUrl}/tree/main/${repo.directory}/src`;
       }
 
@@ -191,10 +205,39 @@ const buildDev = (customConfig = {}) => {
         : undefined;
 
       const output = { $meta: meta, schemas, examples };
+
+      // Multi-export packages ship examples per export, keyed by export name
+      if (devModule.exportExamples) {
+        const exportExamples = toSerializable(devModule.exportExamples);
+        if (exportExamples && Object.keys(exportExamples).length > 0)
+          output.exportExamples = exportExamples;
+      }
+
+      // Multi-export packages ship schemas per export, same shape as schemas
+      if (devModule.exportSchemas) {
+        const exportSchemas = toSerializable(devModule.exportSchemas);
+        if (exportSchemas && Object.keys(exportSchemas).length > 0)
+          output.exportSchemas = exportSchemas;
+      }
+
       if (hints && Object.keys(hints).length > 0) {
         output.hints = hints;
       }
 
+      // Published metadata must never carry a real credential: fail the build
+      assertNoPublishedCredentials(pkg.name, {
+        examples: output.examples,
+        exportExamples: output.exportExamples,
+        hints: output.hints,
+      });
+
+      // Per-export maps match walkerOS.exports; step packages ship settings
+      assertExportMaps(pkg.name, walkerOS.exports, {
+        exportExamples: output.exportExamples,
+        exportSchemas: output.exportSchemas,
+      });
+      assertStepSettings(pkg.name, walkerOS.type, schemas);
+
       // Validate
       if (Object.keys(schemas).length === 0) {
         console.warn('[buildDev] Warning: schemas is empty');