@pikku/inspector 0.12.49 → 0.12.52

package/src/utils/schema-generator.ts

@@ -1,6 +1,12 @@
 import * as ts from 'typescript'
 import { createHash } from 'crypto'
-import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs'
+import {
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  statSync,
+  writeFileSync,
+} from 'fs'
 import { dirname, join, resolve } from 'path'
 import { createGenerator, RootlessError } from 'ts-json-schema-generator'
 import { register } from 'tsx/esm/api'
@@ -75,6 +81,7 @@ let cachedParsedConfig: ts.ParsedCommandLine | undefined
 let cachedTsconfigPath: string | undefined
 let cachedCustomTypesContent: string | undefined
 let cachedTSSchemas: Record<string, JSONValue> | undefined
+let cachedTSSchemaDeps: SchemaDep[] | undefined
 
 const SCHEMA_CACHE_VERSION = 1
 
@@ -118,32 +125,102 @@ function schemaCacheFile(cacheDir: string): string {
   return join(cacheDir, 'ts-schemas.json')
 }
 
-function readDiskTSSchemas(
+/** One file the generated schemas were derived from, as of when they were generated. */
+export type SchemaDep = { path: string; mtimeMs: number; size: number }
+
+/**
+ * The files a set of generated schemas was actually derived from.
+ *
+ * The cache key alone cannot answer whether the schemas are stale. It hashes the
+ * *synthesized* custom-types source, but a named type — `KnowledgeBundle`,
+ * `DbSchema`, anything a function declares as its input or output — appears
+ * there only as a name; its definition is resolved out of a project file or a
+ * dependency's `.d.ts`. Edit one of those and the key is unchanged, so a stale
+ * schema is served for a type that no longer looks like that. The failure is
+ * silent and outlives `rm -rf .pikku`, because this cache lives in
+ * `node_modules/.cache`.
+ *
+ * The program is rooted at the virtual file, so its source files are exactly the
+ * transitive closure the schemas depend on — nothing wider, nothing missed.
+ * Default-lib files are skipped: they change only when TypeScript itself does,
+ * which the inspector version in the key already covers.
+ */
+function fingerprintSchemaDeps(program: ts.Program): SchemaDep[] {
+  const deps: SchemaDep[] = []
+  for (const sourceFile of program.getSourceFiles()) {
+    if (program.isSourceFileDefaultLibrary(sourceFile)) continue
+    try {
+      const stat = statSync(sourceFile.fileName)
+      deps.push({
+        path: sourceFile.fileName,
+        mtimeMs: stat.mtimeMs,
+        size: stat.size,
+      })
+    } catch {
+      // The virtual root has no file on disk. Its content is the custom-types
+      // source, which the cache key already hashes.
+    }
+  }
+  return deps
+}
+
+/**
+ * Whether every file the schemas were derived from is still exactly as it was.
+ *
+ * mtime-and-size rather than a content hash: this runs on the cache-hit path,
+ * where the whole point is to answer in milliseconds without building a TS
+ * program. Anything that rewrites a file — a dependency rebuild, a checkout, an
+ * edit — moves its mtime, and a missing file reads as changed, so the answer errs
+ * toward regenerating.
+ */
+function schemaDepsUnchanged(deps: unknown): boolean {
+  if (!Array.isArray(deps)) return false
+  for (const dep of deps as SchemaDep[]) {
+    if (!dep || typeof dep.path !== 'string') return false
+    try {
+      const stat = statSync(dep.path)
+      if (stat.mtimeMs !== dep.mtimeMs || stat.size !== dep.size) return false
+    } catch {
+      return false
+    }
+  }
+  return true
+}
+
+export function readDiskTSSchemas(
   logger: InspectorLogger,
   cacheDir: string,
   key: string
-): Record<string, JSONValue> | null {
+): { schemas: Record<string, JSONValue>; deps: SchemaDep[] } | null {
   const file = schemaCacheFile(cacheDir)
   if (!existsSync(file)) return null
   try {
     const parsed = JSON.parse(readFileSync(file, 'utf-8'))
-    if (parsed?.key === key && parsed.schemas) return parsed.schemas
+    if (parsed?.key !== key || !parsed.schemas) return null
+    if (!schemaDepsUnchanged(parsed.deps)) {
+      logger.debug(
+        'Discarding TS-schema cache: a type it was built from changed'
+      )
+      return null
+    }
+    return { schemas: parsed.schemas, deps: parsed.deps }
   } catch (e) {
     logger.debug(`Ignoring unreadable TS-schema cache: ${(e as Error).message}`)
   }
   return null
 }
 
-function writeDiskTSSchemas(
+export function writeDiskTSSchemas(
   logger: InspectorLogger,
   cacheDir: string,
   key: string,
-  schemas: Record<string, JSONValue>
+  schemas: Record<string, JSONValue>,
+  deps: SchemaDep[]
 ): void {
   const file = schemaCacheFile(cacheDir)
   try {
     mkdirSync(dirname(file), { recursive: true })
-    writeFileSync(file, JSON.stringify({ key, schemas }))
+    writeFileSync(file, JSON.stringify({ key, schemas, deps }))
   } catch (e) {
     logger.debug(`Failed to persist TS-schema cache: ${(e as Error).message}`)
   }
@@ -234,7 +311,7 @@ function generateTSSchemas(
   additionalTypes?: string[],
   additionalProperties: boolean = false,
   generatedZodSchemas?: Record<string, JSONValue>
-): Record<string, JSONValue> {
+): { schemas: Record<string, JSONValue>; deps: SchemaDep[] } {
   const schemasSet = new Set(typesMap.customTypes.keys())
   for (const { inputs, outputs } of Object.values(functionMeta)) {
     const types = [...(inputs || []), ...(outputs || [])]
@@ -277,7 +354,7 @@ function generateTSSchemas(
     (s) => !PRIMITIVE_TYPES.has(s) && !generatedZodSchemas?.[s]
   )
   if (uncoveredSchemas.length === 0) {
-    return {}
+    return { schemas: {}, deps: [] }
   }
   logger.debug(
     `generateTSSchemas needed for ${uncoveredSchemas.length} types: ${uncoveredSchemas.slice(0, 3).join(', ')}${uncoveredSchemas.length > 3 ? '...' : ''}`
@@ -335,7 +412,28 @@ function generateTSSchemas(
     }
   })
 
-  return schemas
+  // Fingerprinted here, while the program is still alive — it is released as soon
+  // as the schemas are out of it.
+  return { schemas, deps: fingerprintSchemaDeps(program) }
+}
+
+/**
+ * Where the RUNTIME value of a schema lives, given where TypeScript resolved it.
+ *
+ * A schema imported from a built workspace package resolves to that package's
+ * declaration file, and importing a `.d.ts` yields a module with no exports at
+ * all — every one of them is a type. The value is in the emitted JS beside it, so
+ * that is what gets imported.
+ *
+ * Falls back to the original path when no sibling JS exists, which keeps the
+ * "could not find exported schema" error pointing at the file the developer
+ * actually wrote.
+ */
+export const schemaRuntimeFile = (sourceFile: string): string => {
+  const match = /\.d\.([cm]?)ts$/.exec(sourceFile)
+  if (!match) return sourceFile
+  const emitted = sourceFile.replace(/\.d\.([cm]?)ts$/, '.$1js')
+  return existsSync(emitted) ? emitted : sourceFile
 }
 
 /**
@@ -472,7 +570,9 @@ async function generateZodSchemas(
 
   // Collect unique source files and batch-import them in parallel
   const uniqueSourceFiles = [
-    ...new Set([...schemaLookup.values()].map((ref) => ref.sourceFile)),
+    ...new Set(
+      [...schemaLookup.values()].map((ref) => schemaRuntimeFile(ref.sourceFile))
+    ),
   ]
   logger.debug(
     `[TIMING] Zod schemas: ${schemaLookup.size} schemas from ${uniqueSourceFiles.length} files`
@@ -492,12 +592,13 @@ async function generateZodSchemas(
   const fallbackSchemas: [string, SchemaRef][] = []
 
   for (const [schemaName, ref] of schemaLookup.entries()) {
-    const mod = importedModules?.get(ref.sourceFile)
+    const runtimeFile = schemaRuntimeFile(ref.sourceFile)
+    const mod = importedModules?.get(runtimeFile)
     if (mod) {
       const zodSchema = mod[ref.variableName]
       if (!zodSchema) {
         errors.push(
-          `Could not find exported schema '${ref.variableName}' in ${ref.sourceFile} for ${schemaName}. Available exports: ${Object.keys(mod).join(', ')}`
+          `Could not find exported schema '${ref.variableName}' in ${runtimeFile} for ${schemaName}. Available exports: ${Object.keys(mod).join(', ')}`
         )
         continue
       }
@@ -531,12 +632,13 @@ async function generateZodSchemas(
       `Falling back to register() import for ${fallbackSchemas.length} schema(s)`
     )
     for (const [schemaName, ref] of fallbackSchemas) {
+      const runtimeFile = schemaRuntimeFile(ref.sourceFile)
       try {
-        const module = await importWithRegister(ref.sourceFile)
+        const module = await importWithRegister(runtimeFile)
         const zodSchema = module[ref.variableName]
         if (!zodSchema) {
           errors.push(
-            `Could not find exported schema '${ref.variableName}' in ${ref.sourceFile} for ${schemaName}. Available exports: ${Object.keys(module).join(', ')}`
+            `Could not find exported schema '${ref.variableName}' in ${runtimeFile} for ${schemaName}. Available exports: ${Object.keys(module).join(', ')}`
           )
           continue
         }
@@ -595,8 +697,15 @@ export async function generateAllSchemas(
     requiredTypes
   )
 
-  // Fast path: same process, types unchanged — reuse the in-memory result.
-  if (cachedTSSchemas && cachedCustomTypesContent === customTypesContent) {
+  // Fast path: same process, types unchanged — reuse the in-memory result. The
+  // dep check is what makes this correct under `pikku dev`, where the process
+  // outlives the edits: the custom-types source is identical whenever a named
+  // type changes shape without changing its name.
+  if (
+    cachedTSSchemas &&
+    cachedCustomTypesContent === customTypesContent &&
+    schemaDepsUnchanged(cachedTSSchemaDeps)
+  ) {
     logger.debug('Reusing cached TS schemas (types unchanged)')
     return { ...cachedTSSchemas, ...zodSchemas }
   }
@@ -612,12 +721,13 @@ export async function generateAllSchemas(
     if (disk) {
       logger.debug('Reusing on-disk TS schemas (types unchanged across runs)')
       cachedCustomTypesContent = customTypesContent
-      cachedTSSchemas = disk
-      return { ...disk, ...zodSchemas }
+      cachedTSSchemas = disk.schemas
+      cachedTSSchemaDeps = disk.deps
+      return { ...disk.schemas, ...zodSchemas }
     }
   }
 
-  const tsSchemas = generateTSSchemas(
+  const { schemas: tsSchemas, deps: tsSchemaDeps } = generateTSSchemas(
     logger,
     config.tsconfig,
     customTypesContent,
@@ -639,9 +749,16 @@ export async function generateAllSchemas(
 
   cachedCustomTypesContent = customTypesContent
   cachedTSSchemas = tsSchemas
+  cachedTSSchemaDeps = tsSchemaDeps
 
   if (config.cacheDir && cacheKey) {
-    writeDiskTSSchemas(logger, config.cacheDir, cacheKey, tsSchemas)
+    writeDiskTSSchemas(
+      logger,
+      config.cacheDir,
+      cacheKey,
+      tsSchemas,
+      tsSchemaDeps
+    )
   }
 
   return { ...tsSchemas, ...zodSchemas }

package/src/utils/secret-alias-services.test.ts

@@ -0,0 +1,146 @@
+import { describe, test } from 'node:test'
+import assert from 'node:assert/strict'
+import ts from 'typescript'
+import {
+  findSecretAliasServices,
+  isSecretServiceType,
+} from './secret-alias-services.js'
+
+const SECRET_SERVICE = `
+interface SecretService {
+  getSecret(key: string): Promise<string>
+  hasSecret(key: string): Promise<boolean>
+  setSecret(key: string, value: unknown): Promise<void>
+  deleteSecret(key: string): Promise<void>
+  getSecrets(keys: string[]): Promise<Record<string, unknown>>
+}
+`
+
+const singletonServicesTypeOf = (source: string) => {
+  const fileName = '/services.ts'
+  const host: ts.CompilerHost = {
+    ...ts.createCompilerHost({}),
+    getSourceFile: (name, languageVersion) =>
+      name === fileName
+        ? ts.createSourceFile(name, source, languageVersion, true)
+        : undefined,
+    fileExists: (name) => name === fileName,
+    readFile: (name) => (name === fileName ? source : undefined),
+  }
+  const program = ts.createProgram([fileName], { noLib: true }, host)
+  const checker = program.getTypeChecker()
+  const sourceFile = program.getSourceFile(fileName)!
+  const declaration = sourceFile.statements.find(
+    (statement): statement is ts.InterfaceDeclaration =>
+      ts.isInterfaceDeclaration(statement) &&
+      statement.name.text === 'SingletonServices'
+  )!
+  return {
+    checker,
+    type: checker.getTypeAtLocation(declaration),
+  }
+}
+
+describe('findSecretAliasServices', () => {
+  test('finds a secret service exposed under another name', () => {
+    const { type, checker } = singletonServicesTypeOf(`
+      ${SECRET_SERVICE}
+      interface SingletonServices {
+        secrets: SecretService
+        cfg: SecretService
+      }
+    `)
+    assert.deepEqual(findSecretAliasServices(type, checker), ['cfg'])
+  })
+
+  test('never reports `secrets` itself', () => {
+    const { type, checker } = singletonServicesTypeOf(`
+      ${SECRET_SERVICE}
+      interface SingletonServices {
+        secrets: SecretService
+      }
+    `)
+    assert.deepEqual(findSecretAliasServices(type, checker), [])
+  })
+
+  test('leaves ordinary services alone', () => {
+    const { type, checker } = singletonServicesTypeOf(`
+      ${SECRET_SERVICE}
+      interface Logger { info(message: string): void }
+      interface SingletonServices {
+        secrets: SecretService
+        logger: Logger
+      }
+    `)
+    assert.deepEqual(findSecretAliasServices(type, checker), [])
+  })
+
+  test('is not fooled by a service that only has getSecret', () => {
+    const { type, checker } = singletonServicesTypeOf(`
+      ${SECRET_SERVICE}
+      interface Partialish { getSecret(key: string): Promise<string> }
+      interface SingletonServices {
+        secrets: SecretService
+        partialish: Partialish
+      }
+    `)
+    assert.deepEqual(findSecretAliasServices(type, checker), [])
+  })
+
+  test('sees through an optional service', () => {
+    const { type, checker } = singletonServicesTypeOf(`
+      ${SECRET_SERVICE}
+      interface SingletonServices {
+        secrets: SecretService
+        cfg?: SecretService
+      }
+    `)
+    assert.deepEqual(findSecretAliasServices(type, checker), ['cfg'])
+  })
+
+  test('finds a subtype that adds methods', () => {
+    const { type, checker } = singletonServicesTypeOf(`
+      ${SECRET_SERVICE}
+      interface Scoped extends SecretService { getAllStatus(): Promise<void> }
+      interface SingletonServices {
+        secrets: SecretService
+        scoped: Scoped
+      }
+    `)
+    assert.deepEqual(findSecretAliasServices(type, checker), ['scoped'])
+  })
+
+  test('reports every alias, sorted', () => {
+    const { type, checker } = singletonServicesTypeOf(`
+      ${SECRET_SERVICE}
+      interface SingletonServices {
+        secrets: SecretService
+        zed: SecretService
+        alpha: SecretService
+      }
+    `)
+    assert.deepEqual(findSecretAliasServices(type, checker), ['alpha', 'zed'])
+  })
+})
+
+describe('isSecretServiceType', () => {
+  test('refuses a type whose members are not callable', () => {
+    const { type, checker } = singletonServicesTypeOf(`
+      ${SECRET_SERVICE}
+      interface Lookalike {
+        getSecret: string
+        hasSecret: string
+        setSecret: string
+        deleteSecret: string
+        getSecrets: string
+      }
+      interface SingletonServices { lookalike: Lookalike }
+    `)
+    const property = type.getProperty('lookalike')!
+    const propertyType = checker.getTypeOfSymbolAtLocation(
+      property,
+      property.valueDeclaration!
+    )
+    assert.equal(isSecretServiceType(propertyType, checker), false)
+  })
+})

package/src/utils/secret-alias-services.ts

@@ -0,0 +1,58 @@
+import type ts from 'typescript'
+
+const SECRET_SERVICE_METHODS = [
+  'getSecret',
+  'hasSecret',
+  'setSecret',
+  'deleteSecret',
+  'getSecrets',
+]
+
+/** Whether a type structurally satisfies `SecretService`. */
+export const isSecretServiceType = (
+  type: ts.Type,
+  checker: ts.TypeChecker
+): boolean => {
+  const nonNullable = checker.getNonNullableType(type)
+  return SECRET_SERVICE_METHODS.every((method) => {
+    const property = nonNullable.getProperty(method)
+    if (!property) {
+      return false
+    }
+    const declaration = property.valueDeclaration ?? property.declarations?.[0]
+    if (!declaration) {
+      return false
+    }
+    const propertyType = checker.getTypeOfSymbolAtLocation(
+      property,
+      declaration
+    )
+    return propertyType.getCallSignatures().length > 0
+  })
+}
+
+/**
+ * Singleton service names, other than `secrets`, whose type is a
+ * `SecretService`.
+ */
+export const findSecretAliasServices = (
+  singletonServicesType: ts.Type,
+  checker: ts.TypeChecker
+): string[] => {
+  const aliases: string[] = []
+  for (const property of singletonServicesType.getProperties()) {
+    const name = property.getName()
+    if (name === 'secrets') {
+      continue
+    }
+    const declaration = property.valueDeclaration ?? property.declarations?.[0]
+    if (!declaration) {
+      continue
+    }
+    const type = checker.getTypeOfSymbolAtLocation(property, declaration)
+    if (isSecretServiceType(type, checker)) {
+      aliases.push(name)
+    }
+  }
+  return aliases.sort()
+}

package/src/utils/secret-brokers.test.ts

@@ -0,0 +1,53 @@
+import { describe, test } from 'node:test'
+import assert from 'node:assert/strict'
+import { isSecretBrokerFunction } from './secret-brokers.js'
+import { computeSecretBrokers } from './post-process.js'
+import type { InspectorState } from '../types.js'
+
+describe('isSecretBrokerFunction', () => {
+  test('admits the console secret-admin functions', () => {
+    assert.ok(isSecretBrokerFunction('pikkuConsoleGetSecret'))
+    assert.ok(isSecretBrokerFunction('pikkuConsoleSetSecret'))
+    assert.ok(isSecretBrokerFunction('pikkuConsoleHasSecret'))
+  })
+
+  test('admits them when namespaced by an addon', () => {
+    assert.ok(isSecretBrokerFunction('pikku:pikkuConsoleGetSecret'))
+  })
+
+  test('admits nothing else', () => {
+    assert.equal(isSecretBrokerFunction('pikkuConsoleGetVariable'), false)
+    assert.equal(isSecretBrokerFunction('httpRequest'), false)
+    assert.equal(isSecretBrokerFunction('myFunction'), false)
+  })
+
+  test('is not fooled by a name that merely contains a broker name', () => {
+    assert.equal(isSecretBrokerFunction('notPikkuConsoleGetSecret'), false)
+    assert.equal(isSecretBrokerFunction('pikkuConsoleGetSecretly'), false)
+  })
+})
+
+describe('computeSecretBrokers', () => {
+  const stateWith = (meta: Record<string, any>) =>
+    ({ functions: { meta } }) as unknown as InspectorState
+
+  test('flags a broker so the runner keeps its secrets', () => {
+    const state = stateWith({
+      pikkuConsoleGetSecret: { pikkuFuncId: 'pikkuConsoleGetSecret' },
+    })
+    computeSecretBrokers(state)
+    assert.equal(state.functions.meta.pikkuConsoleGetSecret!.secretBroker, true)
+  })
+
+  test('leaves an ordinary function unflagged', () => {
+    const state = stateWith({ createTodo: { pikkuFuncId: 'createTodo' } })
+    computeSecretBrokers(state)
+    assert.equal(state.functions.meta.createTodo!.secretBroker, undefined)
+  })
+
+  test('falls back to the meta key when there is no pikkuFuncId', () => {
+    const state = stateWith({ pikkuConsoleSetSecret: {} })
+    computeSecretBrokers(state)
+    assert.equal(state.functions.meta.pikkuConsoleSetSecret!.secretBroker, true)
+  })
+})

package/src/utils/secret-brokers.ts

@@ -0,0 +1,11 @@
+/** The console's secret administration functions, and nothing else. */
+const SECRET_BROKER_FUNCTIONS = new Set([
+  'pikkuConsoleSetSecret',
+  'pikkuConsoleGetSecret',
+  'pikkuConsoleHasSecret',
+])
+
+/** Whether this function keeps the full `SecretService`. */
+export const isSecretBrokerFunction = (functionId: string): boolean =>
+  SECRET_BROKER_FUNCTIONS.has(functionId) ||
+  SECRET_BROKER_FUNCTIONS.has(functionId.split(':').pop() ?? '')

package/src/utils/serialize-inspector-state.ts

@@ -1,3 +1,4 @@
+import type { SecretUsage } from './extract-secret-usage.js'
 import type { JSONValue } from '@pikku/core'
 import type {
   InspectorDiagnostic,
@@ -201,6 +202,7 @@ export interface SerializableInspectorState {
   secrets: {
     definitions: InspectorState['secrets']['definitions']
     files: string[]
+    usage: Record<string, SecretUsage>
   }
   credentials: {
     definitions: InspectorState['credentials']['definitions']
@@ -210,6 +212,14 @@ export interface SerializableInspectorState {
     definitions: InspectorState['scopes']['definitions']
     files: string[]
   }
+  systemRoles: {
+    definitions: InspectorState['systemRoles']['definitions']
+    files: string[]
+  }
+  personas: {
+    definitions: InspectorState['personas']['definitions']
+    files: string[]
+  }
   variables: {
     definitions: InspectorState['variables']['definitions']
     files: string[]
@@ -416,6 +426,7 @@ export function serializeInspectorState(
     secrets: {
       definitions: state.secrets.definitions,
       files: Array.from(state.secrets.files),
+      usage: Object.fromEntries(state.secrets.usage),
     },
     credentials: {
       definitions: state.credentials.definitions,
@@ -425,6 +436,14 @@ export function serializeInspectorState(
       definitions: state.scopes.definitions,
       files: Array.from(state.scopes.files),
     },
+    systemRoles: {
+      definitions: state.systemRoles.definitions,
+      files: Array.from(state.systemRoles.files),
+    },
+    personas: {
+      definitions: state.personas.definitions,
+      files: Array.from(state.personas.files),
+    },
     variables: {
       definitions: state.variables.definitions,
       files: Array.from(state.variables.files),
@@ -611,6 +630,7 @@ export function deserializeInspectorState(
     secrets: {
       definitions: data.secrets?.definitions || [],
       files: new Set(data.secrets?.files || []),
+      usage: new Map(Object.entries(data.secrets?.usage || {})),
     },
     credentials: {
       definitions: data.credentials?.definitions || [],
@@ -620,6 +640,14 @@ export function deserializeInspectorState(
       definitions: data.scopes?.definitions || [],
       files: new Set(data.scopes?.files || []),
     },
+    systemRoles: {
+      definitions: data.systemRoles?.definitions || [],
+      files: new Set(data.systemRoles?.files || []),
+    },
+    personas: {
+      definitions: data.personas?.definitions || [],
+      files: new Set(data.personas?.files || []),
+    },
     variables: {
       definitions: data.variables?.definitions || [],
       files: new Set(data.variables?.files || []),

package/src/utils/serialize-middleware-groups-meta.ts

@@ -13,6 +13,9 @@ const serializeGroupMap = (
       count: meta.count,
       instanceIds: meta.instanceIds,
       isFactory: meta.isFactory,
+      ...(meta.additionalRegistrations && {
+        additionalRegistrations: meta.additionalRegistrations,
+      }),
     }
   }
   return result