@pikku/inspector 0.12.43 → 0.12.44

package/src/add/add-workflow-sleep-expression.test.ts

@@ -0,0 +1,97 @@
+import { strict as assert } from 'assert'
+import { describe, test } from 'node:test'
+import { mkdtemp, rm, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { inspect } from '../inspector.js'
+import type { InspectorLogger } from '../types.js'
+
+const STEPS = [
+  "import { pikkuSessionlessFunc } from '@pikku/core'",
+  "export const notify = pikkuSessionlessFunc({ func: async () => ({ id: 'n' }) })",
+].join('\n')
+
+async function run(body: string) {
+  const diags: string[] = []
+  const logger = {
+    debug: () => {},
+    info: () => {},
+    warn: () => {},
+    error: () => {},
+    diagnostic: (...a: any[]) => diags.push(a.map(String).join(' ')),
+    critical: (...a: any[]) => diags.push(a.map(String).join(' ')),
+    hasCriticalErrors: () => false,
+  } as unknown as InspectorLogger
+
+  const rootDir = await mkdtemp(join(tmpdir(), 'pikku-sleep-'))
+  const stepFile = join(rootDir, 'w.steps.ts')
+  const wfFile = join(rootDir, 'w.workflow.ts')
+  await writeFile(stepFile, STEPS)
+  await writeFile(
+    wfFile,
+    [
+      "import { pikkuWorkflowFunc } from '@pikku/core/workflow'",
+      'export const wf = pikkuWorkflowFunc(async (_, data: any, { workflow }) => {',
+      body,
+      '})',
+    ].join('\n')
+  )
+  try {
+    const state: any = await inspect(logger, [stepFile, wfFile], { rootDir })
+    const gm = state.workflows.graphMeta
+    const graph = (gm instanceof Map ? [...gm.values()] : Object.values(gm))[0]
+    return { graph: graph as any, diags }
+  } finally {
+    await rm(rootDir, { recursive: true, force: true })
+  }
+}
+
+describe('sleep with a duration that is not a literal', () => {
+  test('a variable duration inside a loop is kept, not rejected', async () => {
+    const { graph, diags } = await run(
+      [
+        '  for (const interval of data.intervals) {',
+        '    await workflow.sleep(`Wait ${interval}`, interval)',
+        "    await workflow.do('Notify', 'notify', {})",
+        '  }',
+        '  return { sent: true }',
+      ].join('\n')
+    )
+
+    assert.deepEqual(
+      diags.filter((d) => d.includes('sleep')),
+      [],
+      'a duration only known at runtime is legal — the closure evaluates it'
+    )
+
+    const sleepNode = Object.values(graph?.nodes ?? {}).find(
+      (n: any) => n.flow === 'sleep'
+    ) as any
+    assert.ok(sleepNode, 'the sleep must still appear in the graph')
+    assert.equal(
+      sleepNode.expression,
+      'interval',
+      'its source text is recorded so the graph can show what it waits on'
+    )
+  })
+
+  test('a literal duration still round-trips as a value', async () => {
+    const { graph, diags } = await run(
+      [
+        "  await workflow.sleep('Wait', '5s')",
+        '  return { sent: true }',
+      ].join('\n')
+    )
+
+    assert.deepEqual(diags, [])
+    const sleepNode = Object.values(graph?.nodes ?? {}).find(
+      (n: any) => n.flow === 'sleep'
+    ) as any
+    assert.equal(sleepNode?.duration, '5s')
+    assert.equal(
+      sleepNode?.expression,
+      undefined,
+      'a literal is a value, not an expression'
+    )
+  })
+})

package/src/add/add-workflow.ts

@@ -73,8 +73,8 @@ function hasInlineSteps(steps: WorkflowStepMeta[]): boolean {
         if (c.steps && hasInlineSteps(c.steps)) return true
       }
       if (step.defaultSteps && hasInlineSteps(step.defaultSteps)) return true
-    } else if (step.type === 'fanout' && step.child) {
-      if (hasInlineSteps([step.child])) return true
+    } else if (step.type === 'fanout' && step.body) {
+      if (hasInlineSteps(step.body)) return true
     } else if (step.type === 'parallel' && step.children) {
       if (hasInlineSteps(step.children)) return true
     }
@@ -92,6 +92,11 @@ export function collectInvokedRPCs(
   for (const step of steps) {
     if (step.type === 'rpc' && step.rpcName) {
       rpcs.add(step.rpcName)
+      // The compensation handler is invoked at runtime too, so it has to be
+      // wired like any other step.
+      if (step.options?.onError) {
+        rpcs.add(step.options.onError)
+      }
     } else if (step.type === 'branch') {
       for (const branch of step.branches) {
         collectInvokedRPCs(branch.steps, rpcs)
@@ -102,8 +107,8 @@ export function collectInvokedRPCs(
         if (c.steps) collectInvokedRPCs(c.steps, rpcs)
       }
       if (step.defaultSteps) collectInvokedRPCs(step.defaultSteps, rpcs)
-    } else if (step.type === 'fanout' && step.child) {
-      collectInvokedRPCs([step.child], rpcs)
+    } else if (step.type === 'fanout' && step.body) {
+      collectInvokedRPCs(step.body, rpcs)
     } else if (step.type === 'parallel' && step.children) {
       collectInvokedRPCs(step.children, rpcs)
     }

package/src/add/config-value-metadata.test.ts

@@ -0,0 +1,169 @@
+import { strict as assert } from 'assert'
+import { describe, test } from 'node:test'
+import { mkdtemp, rm, writeFile } from 'node:fs/promises'
+import { join } from 'node:path'
+import { fileURLToPath } from 'node:url'
+import { inspect } from '../inspector.js'
+import type { ErrorCode } from '../error-codes.js'
+import type { InspectorLogger } from '../types.js'
+
+// Fixtures must sit inside the package so `zod` resolves — the schema-vendor
+// check rejects a schema it cannot trace to a supported library, and the
+// definition is then never recorded at all.
+const fixtureRoot = join(
+  fileURLToPath(new URL('.', import.meta.url)),
+  '__fixtures'
+)
+
+const makeLogger = (criticals: Array<{ code: ErrorCode; message: string }>) =>
+  ({
+    debug: () => {},
+    info: () => {},
+    warn: () => {},
+    error: () => {},
+    diagnostic: ({ code, message }) => {
+      criticals.push({ code, message })
+    },
+    critical: (code: ErrorCode, message: string) => {
+      criticals.push({ code, message })
+    },
+    hasCriticalErrors: () => criticals.length > 0,
+  }) satisfies InspectorLogger
+
+/**
+ * `docsUrl` is only useful if it survives inspection — a field declared on the
+ * wiring but dropped by the inspector is invisible to every consumer
+ * downstream, which is exactly how `displayName` went unused.
+ */
+describe('config value metadata (docsUrl)', () => {
+  test('carries docsUrl from wireSecret into the definition', async () => {
+    const rootDir = await mkdtemp(`${fixtureRoot}-secret-`)
+    const file = join(rootDir, 'secret.ts')
+
+    await writeFile(
+      file,
+      [
+        "import { wireSecret } from '@pikku/core'",
+        "import { z } from 'zod'",
+        'export const StripeKeySchema = z.string()',
+        'wireSecret({',
+        "  name: 'stripeKey',",
+        "  displayName: 'Stripe Secret Key',",
+        "  secretId: 'STRIPE_SECRET_KEY',",
+        "  docsUrl: 'https://dashboard.stripe.com/apikeys',",
+        '  schema: StripeKeySchema,',
+        '})',
+      ].join('\n')
+    )
+
+    const criticals: Array<{ code: ErrorCode; message: string }> = []
+    try {
+      const state = await inspect(makeLogger(criticals), [file], { rootDir })
+      const def = state.secrets.definitions.find(
+        (d) => d.secretId === 'STRIPE_SECRET_KEY'
+      )
+      assert.ok(def, 'expected the secret definition to be inspected')
+      assert.equal(def!.docsUrl, 'https://dashboard.stripe.com/apikeys')
+    } finally {
+      await rm(rootDir, { recursive: true, force: true })
+    }
+  })
+
+  test('leaves docsUrl undefined when not declared', async () => {
+    const rootDir = await mkdtemp(`${fixtureRoot}-bare-`)
+    const file = join(rootDir, 'secret.ts')
+
+    await writeFile(
+      file,
+      [
+        "import { wireSecret } from '@pikku/core'",
+        "import { z } from 'zod'",
+        'export const ApiKeySchema = z.string()',
+        'wireSecret({',
+        "  name: 'apiKey',",
+        "  displayName: 'API Key',",
+        "  secretId: 'API_KEY',",
+        '  schema: ApiKeySchema,',
+        '})',
+      ].join('\n')
+    )
+
+    const criticals: Array<{ code: ErrorCode; message: string }> = []
+    try {
+      const state = await inspect(makeLogger(criticals), [file], { rootDir })
+      const def = state.secrets.definitions.find(
+        (d) => d.secretId === 'API_KEY'
+      )
+      assert.ok(def, 'expected the secret definition to be inspected')
+      assert.equal(def!.docsUrl, undefined)
+    } finally {
+      await rm(rootDir, { recursive: true, force: true })
+    }
+  })
+
+  test('carries docsUrl from wireVariable into the definition', async () => {
+    const rootDir = await mkdtemp(`${fixtureRoot}-variable-`)
+    const file = join(rootDir, 'variable.ts')
+
+    await writeFile(
+      file,
+      [
+        "import { wireVariable } from '@pikku/core'",
+        "import { z } from 'zod'",
+        'export const ConsoleUrlSchema = z.string()',
+        'wireVariable({',
+        "  name: 'consoleUrl',",
+        "  displayName: 'Console URL',",
+        "  variableId: 'CONSOLE_URL',",
+        "  docsUrl: 'https://example.com/docs/console-url',",
+        '  schema: ConsoleUrlSchema,',
+        '})',
+      ].join('\n')
+    )
+
+    const criticals: Array<{ code: ErrorCode; message: string }> = []
+    try {
+      const state = await inspect(makeLogger(criticals), [file], { rootDir })
+      const def = state.variables.definitions.find(
+        (d) => d.variableId === 'CONSOLE_URL'
+      )
+      assert.ok(def, 'expected the variable definition to be inspected')
+      assert.equal(def!.docsUrl, 'https://example.com/docs/console-url')
+    } finally {
+      await rm(rootDir, { recursive: true, force: true })
+    }
+  })
+
+  test('carries docsUrl from wireCredential into the definition', async () => {
+    const rootDir = await mkdtemp(`${fixtureRoot}-credential-`)
+    const file = join(rootDir, 'credential.ts')
+
+    await writeFile(
+      file,
+      [
+        "import { wireCredential } from '@pikku/core'",
+        "import { z } from 'zod'",
+        'export const TokenSchema = z.object({ token: z.string() })',
+        'wireCredential({',
+        "  name: 'githubToken',",
+        "  displayName: 'GitHub Token',",
+        "  type: 'wire',",
+        "  docsUrl: 'https://github.com/settings/tokens',",
+        '  schema: TokenSchema,',
+        '})',
+      ].join('\n')
+    )
+
+    const criticals: Array<{ code: ErrorCode; message: string }> = []
+    try {
+      const state = await inspect(makeLogger(criticals), [file], { rootDir })
+      const def = state.credentials.definitions.find(
+        (d) => d.name === 'githubToken'
+      )
+      assert.ok(def, 'expected the credential definition to be inspected')
+      assert.equal(def!.docsUrl, 'https://github.com/settings/tokens')
+    } finally {
+      await rm(rootDir, { recursive: true, force: true })
+    }
+  })
+})

package/src/utils/schema-generator.test.ts

@@ -0,0 +1,164 @@
+import { test, describe, before, after } from 'node:test'
+import assert from 'node:assert/strict'
+import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'fs'
+import { tmpdir } from 'os'
+import { join } from 'path'
+
+import { getInitialInspectorState } from '../inspector.js'
+import { generateAllSchemas } from './schema-generator.js'
+
+const debugLines: string[] = []
+const logger = {
+  debug(message: string) {
+    debugLines.push(message)
+  },
+  info() {},
+  warn() {},
+  error() {},
+} as any
+
+const UNRELATED_FILE_COUNT = 300
+
+let projectDir: string
+
+before(() => {
+  projectDir = mkdtempSync(join(tmpdir(), 'pikku-schema-gen-'))
+  mkdirSync(join(projectDir, 'src'))
+  writeFileSync(
+    join(projectDir, 'tsconfig.json'),
+    JSON.stringify({
+      compilerOptions: {
+        target: 'ESNext',
+        module: 'NodeNext',
+        moduleResolution: 'NodeNext',
+        strict: true,
+        skipLibCheck: true,
+      },
+      include: ['src/**/*.ts'],
+    })
+  )
+  writeFileSync(
+    join(projectDir, 'src', 'types.ts'),
+    'export type Thing = { id: string; count: number }\n'
+  )
+  // Files the tsconfig picks up but no generated custom type references. They
+  // stand in for the bulk of a real project, which the schema program has no
+  // reason to load.
+  for (let i = 0; i < UNRELATED_FILE_COUNT; i += 1) {
+    writeFileSync(
+      join(projectDir, 'src', `unrelated-${i}.ts`),
+      `export type Unrelated${i} = { a: string; b: number; c: boolean }\n`
+    )
+  }
+})
+
+after(() => {
+  rmSync(projectDir, { recursive: true, force: true })
+})
+
+describe('generateAllSchemas', () => {
+  // NOTE: this must be the first test in the file. The module-level caches are
+  // per-process, so only the first call actually builds a program — a later call
+  // reuses it and would measure a few MB no matter what.
+  //
+  // The schema generator builds a second ts.Program over the whole tsconfig —
+  // on a real tree that is ~2.5k files and ~600MB, and it used to be pinned at
+  // module scope for the life of the process. Combined with the main inspector
+  // program it pushed peak heap past 2GB and OOM'd `pikku all` (#982).
+  //
+  // Only the program is released; cachedTSSchemas (the small result cache that
+  // powers the same-process fast path) is deliberately kept.
+  test('does not retain the schema ts.Program after returning', async (t) => {
+    if (typeof global.gc !== 'function') {
+      t.skip('requires --expose-gc')
+      return
+    }
+
+    const state = getInitialInspectorState(projectDir) as any
+    // A distinct type keeps this off the cachedTSSchemas fast path, so the
+    // program is genuinely rebuilt (and therefore genuinely retainable) here.
+    state.functions.typesMap.addCustomType(
+      'RetentionProbeInput',
+      '{ probe: string; n: number }',
+      []
+    )
+
+    global.gc!()
+    const before = process.memoryUsage().heapUsed
+
+    await generateAllSchemas(
+      logger,
+      { tsconfig: join(projectDir, 'tsconfig.json') },
+      state
+    )
+
+    global.gc!()
+    const retainedMB = (process.memoryUsage().heapUsed - before) / 1024 / 1024
+
+    // A retained program (with its SourceFiles and TypeChecker) measures tens of
+    // MB even for this one-file fixture; the schemas themselves are a few KB.
+    assert.ok(
+      retainedMB < 10,
+      `expected the schema program to be released, but ${retainedMB.toFixed(1)}MB is still retained after GC`
+    )
+  })
+
+  // The virtual file explicitly imports every type it needs, so it is a complete
+  // root on its own. Rooting the program at the whole tsconfig instead made it
+  // load the entire project (2572 files on a real tree vs 870), for no gain in
+  // what could be resolved (#982).
+  test('scopes the schema program to the virtual file import closure', async () => {
+    debugLines.length = 0
+
+    const state = getInitialInspectorState(projectDir) as any
+    state.functions.typesMap.addCustomType(
+      'ScopingProbeInput',
+      '{ scope: string }',
+      []
+    )
+
+    await generateAllSchemas(
+      logger,
+      { tsconfig: join(projectDir, 'tsconfig.json') },
+      state
+    )
+
+    const line = debugLines.find((l) => l.includes('Created schema program'))
+    assert.ok(
+      line,
+      `expected a "Created schema program" debug line, got: ${debugLines.join(' | ')}`
+    )
+
+    const fileCount = Number(line!.match(/(\d+) files/)?.[1])
+    assert.ok(
+      Number.isFinite(fileCount),
+      `could not parse a file count out of: ${line}`
+    )
+    // The virtual file plus types.ts plus the default lib. Rooting at the whole
+    // tsconfig would pull in all UNRELATED_FILE_COUNT files on top of that.
+    assert.ok(
+      fileCount < UNRELATED_FILE_COUNT,
+      `expected the schema program to skip the ${UNRELATED_FILE_COUNT} unreferenced files, but it loaded ${fileCount}`
+    )
+  })
+
+  test('produces a schema for a custom type', async () => {
+    const state = getInitialInspectorState(projectDir) as any
+    state.functions.typesMap.addCustomType(
+      'ThingInput',
+      '{ id: string; count: number }',
+      []
+    )
+
+    const schemas = await generateAllSchemas(
+      logger,
+      { tsconfig: join(projectDir, 'tsconfig.json') },
+      state
+    )
+
+    assert.ok(
+      Object.keys(schemas).length > 0,
+      'expected at least one generated schema'
+    )
+  })
+})

package/src/utils/schema-generator.ts

@@ -87,7 +87,9 @@ const inspectorVersion: string = (() => {
   try {
     const pkgUrl = new URL('../../package.json', import.meta.url)
     const pkg = JSON.parse(readFileSync(pkgUrl, 'utf-8'))
-    return typeof pkg.version === 'string' ? pkg.version : `v${SCHEMA_CACHE_VERSION}`
+    return typeof pkg.version === 'string'
+      ? pkg.version
+      : `v${SCHEMA_CACHE_VERSION}`
   } catch {
     return `v${SCHEMA_CACHE_VERSION}`
   }
@@ -98,7 +100,10 @@ const inspectorVersion: string = (() => {
 // inspector version (schema-format changes ship with a version bump).
 function tsSchemaCacheKey(
   customTypesContent: string,
-  config: { schemasFromTypes?: string[]; schema?: { additionalProperties?: boolean } }
+  config: {
+    schemasFromTypes?: string[]
+    schema?: { additionalProperties?: boolean }
+  }
 ): string {
   return createHash('sha1')
     .update(`v${SCHEMA_CACHE_VERSION}\0`)
@@ -145,6 +150,7 @@ function writeDiskTSSchemas(
 }
 
 function createProgramWithVirtualFile(
+  logger: InspectorLogger,
   tsconfig: string,
   virtualFilePath: string,
   virtualFileContent: string
@@ -165,7 +171,11 @@ function createProgramWithVirtualFile(
   }
 
   const resolvedVirtualPath = resolve(virtualFilePath)
-  const fileNames = [...cachedParsedConfig.fileNames, resolvedVirtualPath]
+  // The virtual file imports every type it references, so it is a complete root
+  // on its own and TypeScript pulls in exactly the transitive closure it needs.
+  // Rooting at cachedParsedConfig.fileNames instead loaded the whole project
+  // (2572 files vs 870 on a real tree) without widening what could be resolved.
+  const fileNames = [resolvedVirtualPath]
 
   const defaultHost = ts.createCompilerHost(cachedParsedConfig.options)
   const customHost: ts.CompilerHost = {
@@ -200,6 +210,7 @@ function createProgramWithVirtualFile(
     },
   }
 
+  const startProgram = performance.now()
   const program = ts.createProgram(
     fileNames,
     cachedParsedConfig.options,
@@ -207,6 +218,9 @@ function createProgramWithVirtualFile(
     cachedSchemaProgram // reuse previous program for incremental compilation
   )
   cachedSchemaProgram = program
+  logger.debug(
+    `Created schema program in ${(performance.now() - startProgram).toFixed(0)}ms (${program.getSourceFiles().length} files)`
+  )
   return program
 }
 
@@ -274,6 +288,7 @@ function generateTSSchemas(
     '__pikku_virtual_types__.ts'
   )
   const program = createProgramWithVirtualFile(
+    logger,
     tsconfig,
     virtualFilePath,
     customTypesContent
@@ -614,6 +629,14 @@ export async function generateAllSchemas(
     zodSchemas
   )
 
+  // Release the program once the schemas are out of it. It spans the whole
+  // tsconfig (~2.5k files, ~600MB on a large tree) and was previously pinned at
+  // module scope for the life of the process, so it stacked on top of the main
+  // inspector program and pushed `pikku all` past a 2GB heap (#982). The result
+  // cache below is what the same-process fast path actually needs; the program
+  // only bought incremental reuse on a rebuild, at a cost that dwarfs it.
+  cachedSchemaProgram = undefined
+
   cachedCustomTypesContent = customTypesContent
   cachedTSSchemas = tsSchemas