@pikku/inspector 0.12.75 → 0.12.81

package/src/inspector.ts

@@ -1,575 +0,0 @@
-import * as ts from 'typescript'
-import { performance } from 'perf_hooks'
-import { dirname, relative, resolve } from 'path'
-import { visitSetup, visitFunctions, visitRoutes } from './visit.js'
-import { TypesMap } from './types-map.js'
-import type {
-  InspectorState,
-  InspectorLogger,
-  InspectorOptions,
-} from './types.js'
-import { getFilesAndMethods } from './utils/get-files-and-methods.js'
-import { findCommonAncestor } from './utils/find-root-dir.js'
-import { createSourceFileCacheHost } from './source-file-cache.js'
-import { createNestedProjectFilter } from './utils/nested-project-filter.js'
-import {
-  aggregateRequiredServices,
-  stampAuthHandlerServices,
-  validateAgentModels,
-  validateAgentToolReferences,
-  validateAgentScorerReferences,
-  validateSecretOverrides,
-  validateVariableOverrides,
-  validateCredentialOverrides,
-  validateRemoteAddonDependencies,
-  validateRemoteAddonAuth,
-  validateScopeReferences,
-  validateSystemRoleScopes,
-  validatePersonaRoles,
-  computeResolvedIOTypes,
-  computeMiddlewareGroupsMeta,
-  computePermissionsGroupsMeta,
-  computeRequiredSchemas,
-  validateNoSecretAliasServices,
-  validateSecretUsage,
-  computeDiagnostics,
-  validateSchemaReferences,
-  validateSchemaWiringSeparation,
-  validateScenarioServices,
-  validateScenarioSteps,
-  validateWorkflowGraphAddons,
-} from './utils/post-process.js'
-import { validateExposedFunctionsGated } from './utils/validate-exposed-functions-gated.js'
-import { validateTagsResolveToMiddleware } from './utils/validate-tags-resolve-to-middleware.js'
-import { annotateHttpRouteAuth } from './utils/annotate-http-route-auth.js'
-import { generateOpenAPISpec } from './utils/serialize-openapi-json.js'
-import { pikkuState } from '@pikku/core/state'
-import { resolveLatestVersions } from './utils/resolve-versions.js'
-import { finalizeWorkflows } from './utils/workflow/graph/finalize-workflows.js'
-import {
-  finalizeWorkflowHelperTypes,
-  finalizeWorkflowWires,
-} from './utils/workflow/graph/finalize-workflow-wires.js'
-import { generateAllSchemas } from './utils/schema-generator.js'
-import { extractSecretUsage } from './utils/extract-secret-usage.js'
-import {
-  accumulateSurfaceUsage,
-  surfaceUsageArea,
-} from './utils/extract-surface-usage.js'
-import {
-  loadAddonFunctionsMeta,
-  loadAddonSchemas,
-} from './utils/load-addon-functions-meta.js'
-import {
-  computeContractHashes,
-  extractContractsFromMeta,
-  updateManifest,
-  createEmptyManifest,
-  validateContracts,
-} from './utils/contract-hashes.js'
-
-/**
- * Creates an initial/empty inspector state with all required properties initialized
- * @param rootDir - The root directory for the project
- * @returns A fresh InspectorState with empty collections
- */
-export function getInitialInspectorState(rootDir: string): InspectorState {
-  return {
-    rootDir,
-    singletonServicesTypeImportMap: new Map(),
-    wireServicesTypeImportMap: new Map(),
-    userSessionTypeImportMap: new Map(),
-    configTypeImportMap: new Map(),
-    singletonServicesFactories: new Map(),
-    wireServicesFactories: new Map(),
-    wireServicesMeta: new Map(),
-    addonRequiredParentServices: [],
-    addonCreatedServices: [],
-    addonServicesFactorySeen: false,
-    addonServerlessIncompatible: new Map(),
-    configFactories: new Map(),
-    serverLifecycleFactories: new Map(),
-    filesAndMethods: {},
-    filesAndMethodsErrors: new Map(),
-    typesLookup: new Map(),
-    schemaLookup: new Map(),
-    schemas: {},
-    functions: {
-      typesMap: new TypesMap(),
-      meta: {},
-      files: new Map(),
-      approvalDescriptions: {},
-      dynamicImportIds: new Set(),
-    },
-    http: {
-      metaInputTypes: new Map(),
-      meta: {
-        get: {},
-        post: {},
-        put: {},
-        delete: {},
-        head: {},
-        patch: {},
-        options: {},
-      },
-      files: new Set(),
-      routeMiddleware: new Map(),
-    },
-    channels: {
-      files: new Set(),
-      meta: {},
-    },
-    gateways: {
-      meta: {},
-      files: new Set(),
-    },
-    triggers: {
-      meta: {},
-      sourceMeta: {},
-      files: new Set(),
-    },
-    scheduledTasks: {
-      meta: {},
-      files: new Set(),
-    },
-    queueWorkers: {
-      meta: {},
-      files: new Set(),
-    },
-    workflows: {
-      meta: {},
-      files: new Map(),
-      graphMeta: {},
-      graphFiles: new Map(),
-      featureFiles: new Map(),
-      invokedWorkflows: new Set(),
-    },
-    rpc: {
-      internalMeta: {},
-      internalFiles: new Map(),
-      exposedMeta: {},
-      exposedFiles: new Map(),
-      remoteMeta: {},
-      remoteFiles: new Map(),
-      invokedFunctions: new Set(),
-      invokedFunctionsByFile: new Map(),
-      usedAddons: new Set(),
-      wireAddonDeclarations: new Map(),
-      wireAddonFiles: new Set(),
-    },
-    mcpEndpoints: {
-      resourcesMeta: {},
-      toolsMeta: {},
-      promptsMeta: {},
-      files: new Set(),
-    },
-    agents: {
-      agentsMeta: {},
-      files: new Map(),
-      invokedAgentsByFile: new Map(),
-    },
-    scorers: {
-      scorersMeta: {},
-      files: new Map(),
-    },
-    cli: {
-      meta: {
-        programs: {},
-        renderers: {},
-      },
-      files: new Set(),
-    },
-    nodes: {
-      meta: {},
-      files: new Set(),
-    },
-    auth: {
-      providers: [],
-      plugins: [],
-      files: new Set(),
-      definition: null,
-    },
-    secrets: {
-      definitions: [],
-      files: new Set(),
-      usage: new Map(),
-    },
-    surfaceUsage: {},
-    credentials: {
-      definitions: [],
-      files: new Set(),
-    },
-    scopes: {
-      definitions: [],
-      files: new Set(),
-    },
-    systemRoles: {
-      definitions: [],
-      files: new Set(),
-    },
-    personas: {
-      definitions: [],
-      files: new Set(),
-    },
-    variables: {
-      definitions: [],
-      files: new Set(),
-    },
-    manifest: {
-      initial: null,
-      current: null,
-      errors: [],
-    },
-    middleware: {
-      definitions: {},
-      instances: {},
-      tagMiddleware: new Map(),
-    },
-    channelMiddleware: {
-      definitions: {},
-      instances: {},
-      tagMiddleware: new Map(),
-    },
-    agentMiddleware: {
-      definitions: {},
-    },
-    permissions: {
-      definitions: {},
-      instances: {},
-    },
-    serviceAggregation: {
-      requiredServices: new Set(),
-      usedFunctions: new Set(),
-      usedMiddleware: new Set(),
-      usedPermissions: new Set(),
-      allSingletonServices: [],
-      allWireServices: [],
-    },
-    resolvedIOTypes: {},
-    middlewareGroupsMeta: {
-      definitions: {},
-      instances: {},
-      httpGroups: {},
-      tagGroups: {},
-      channelMiddleware: {
-        definitions: {},
-        instances: {},
-        tagGroups: {},
-      },
-    },
-    permissionsGroupsMeta: {
-      definitions: {},
-    },
-    requiredSchemas: new Set(),
-    openAPISpec: null,
-    diagnostics: [],
-    addonFunctions: {},
-    exportedContracts: {
-      http: {},
-      cli: {},
-      channel: {},
-      addonHttp: {},
-      addonCli: {},
-      addonChannel: {},
-    },
-    program: null,
-  }
-}
-
-/**
- * The path mappings the inspector's program has to share with the project.
- *
- * The inspector builds its own program rather than reusing the project's, which
- * is what keeps a cold run affordable — but a project that reaches its generated
- * code through a tsconfig `paths` entry resolves that import through those
- * mappings and nothing else. Without them the factory an inspected function is
- * passed to resolves to `any`, so a function whose input type comes from the
- * factory's contextual type (rather than an explicit generic or a named schema)
- * silently records no input at all.
- *
- * Only resolution keys are copied. Type-inference options such as `strict` stay
- * the inspector's own, because changing those would change the recorded types.
- */
-const readProjectPathMappings = (
-  logger: InspectorLogger,
-  tsconfig: string | undefined
-): ts.CompilerOptions => {
-  if (!tsconfig) {
-    return {}
-  }
-  const configPath = resolve(tsconfig)
-  const configFile = ts.readConfigFile(configPath, ts.sys.readFile)
-  if (configFile.error || !configFile.config) {
-    logger.debug(
-      `Could not read ${configPath} for path mappings; imports that rely on them will not resolve`
-    )
-    return {}
-  }
-  const { options: projectOptions } = ts.parseJsonConfigFileContent(
-    configFile.config,
-    ts.sys,
-    dirname(configPath)
-  )
-  const { baseUrl, paths, rootDirs, pathsBasePath } = projectOptions
-  return {
-    ...(baseUrl !== undefined ? { baseUrl } : {}),
-    ...(paths !== undefined ? { paths } : {}),
-    ...(rootDirs !== undefined ? { rootDirs } : {}),
-    ...(pathsBasePath !== undefined ? { pathsBasePath } : {}),
-  }
-}
-
-export const inspect = async (
-  logger: InspectorLogger,
-  routeFiles: string[],
-  options: InspectorOptions = {}
-): Promise<InspectorState> => {
-  const normalizedRouteFiles = routeFiles.map((file) => resolve(file))
-  const compilerOptions: ts.CompilerOptions = {
-    ...readProjectPathMappings(logger, options.tsconfig),
-    target: ts.ScriptTarget.ESNext,
-    module: ts.ModuleKind.Node16,
-    skipLibCheck: true,
-    skipDefaultLibCheck: true,
-    moduleResolution: ts.ModuleResolutionKind.Node16,
-    types: [],
-    allowJs: false,
-    checkJs: false,
-  }
-  const cpuStart = process.cpuUsage()
-  const startProgram = performance.now()
-  const host = createSourceFileCacheHost(compilerOptions)
-  const program = ts.createProgram(
-    normalizedRouteFiles,
-    compilerOptions,
-    host,
-    options.oldProgram
-  )
-  host.prune(program)
-  const filesReused = host.reused()
-  logger.debug(
-    `Created program in ${(performance.now() - startProgram).toFixed(0)}ms (${normalizedRouteFiles.length} files, ${filesReused} source files reused)`
-  )
-
-  const startChecker = performance.now()
-  const checker = program.getTypeChecker()
-  logger.debug(
-    `Got type checker in ${(performance.now() - startChecker).toFixed(2)}ms`
-  )
-
-  // Use provided rootDir or infer from source files
-  const rootDir = options.rootDir || findCommonAncestor(normalizedRouteFiles)
-
-  const startSourceFiles = performance.now()
-  // node_modules under rootDir (e.g. a locally-installed addon) is a
-  // dependency, not project source — scanning it double-counts the addon's
-  // own application types (CoreConfig/Services/SingletonServices). Nested
-  // pikku projects (a workspace addon at packages/<addon>) are the same case:
-  // TS resolves the node_modules symlink to its realpath under rootDir, so
-  // the node_modules check alone misses them.
-  // Sort by file name so the sweeps populate state in a stable order. The
-  // program's own file order depends on glob + import-graph resolution, which
-  // varies run to run — leaving generated meta keys (and anything serialized
-  // in insertion order) non-reproducible across identical `pikku all` runs.
-  // Safe because function registration is a dedicated pass (visitFunctions)
-  // that completes before any order-sensitive wiring resolution in visitRoutes.
-  const isNestedProjectFile = createNestedProjectFilter(rootDir)
-  const sourceFiles = program
-    .getSourceFiles()
-    .filter(
-      (sf) =>
-        sf.fileName.startsWith(rootDir) &&
-        !sf.fileName.includes('/node_modules/') &&
-        !isNestedProjectFile(sf.fileName)
-    )
-    .sort((a, b) =>
-      a.fileName < b.fileName ? -1 : a.fileName > b.fileName ? 1 : 0
-    )
-  logger.debug(
-    `Got source files in ${(performance.now() - startSourceFiles).toFixed(2)}ms`
-  )
-
-  const state = getInitialInspectorState(rootDir)
-
-  const surfaceAreaCache = new Map<string, string | null>()
-  for (const sourceFile of sourceFiles) {
-    const usage = extractSecretUsage(sourceFile)
-    if (usage.keys.length > 0 || usage.dynamic.length > 0) {
-      state.secrets.usage.set(sourceFile.fileName, usage)
-    }
-    // Counted in the sweep that already has this file open. The public surface
-    // is documentation, and documentation may not cost a project a second pass
-    // over its source.
-    accumulateSurfaceUsage(
-      sourceFile,
-      {
-        area: surfaceUsageArea(sourceFile.fileName, rootDir, surfaceAreaCache),
-        file: relative(rootDir, sourceFile.fileName),
-      },
-      state.surfaceUsage
-    )
-  }
-
-  // First sweep: add all functions
-  const startSetup = performance.now()
-  for (const sourceFile of sourceFiles) {
-    const sourceOptions = { ...options, sourceFile }
-    ts.forEachChild(sourceFile, (child) =>
-      visitSetup(logger, checker, child, state, sourceOptions)
-    )
-  }
-  logger.debug(
-    `Visit setup phase completed in ${(performance.now() - startSetup).toFixed(0)}ms`
-  )
-
-  // Load addon function metadata so wirings can reference addon functions
-  await loadAddonFunctionsMeta(logger, state)
-
-  if (!options.setupOnly) {
-    // Function sweep: register every function before transports/wirings resolve
-    // them, so resolution doesn't depend on source-file order.
-    const startFunctions = performance.now()
-    for (const sourceFile of sourceFiles) {
-      const sourceOptions = { ...options, sourceFile }
-      ts.forEachChild(sourceFile, (child) =>
-        visitFunctions(logger, checker, child, state, sourceOptions)
-      )
-    }
-    logger.debug(
-      `Visit functions phase completed in ${(performance.now() - startFunctions).toFixed(0)}ms`
-    )
-
-    // Transport sweep: add all transports/wirings
-    const startRoutes = performance.now()
-    for (const sourceFile of sourceFiles) {
-      const sourceOptions = { ...options, sourceFile }
-      ts.forEachChild(sourceFile, (child) =>
-        visitRoutes(logger, checker, child, state, sourceOptions)
-      )
-    }
-    logger.debug(
-      `Visit routes phase completed in ${(performance.now() - startRoutes).toFixed(0)}ms`
-    )
-
-    resolveLatestVersions(state, logger)
-
-    if (options.schemaConfig) {
-      const startSchemas = performance.now()
-      state.schemas = await generateAllSchemas(
-        logger,
-        options.schemaConfig,
-        state
-      )
-      logger.debug(
-        `generateAllSchemas took ${(performance.now() - startSchemas).toFixed(0)}ms`
-      )
-      computeContractHashes(
-        state.schemas,
-        state.functions.typesMap,
-        state.functions.meta
-      )
-      computeRequiredSchemas(state, options)
-    }
-
-    // Re-load addon schemas (generateAllSchemas replaces state.schemas)
-    await loadAddonSchemas(logger, state)
-
-    // After the addon merge, not before: an addon supplies its own schemas here,
-    // and checking earlier would report every one of them as unresolved.
-    if (options.schemaConfig) {
-      validateSchemaReferences(logger, state)
-    }
-
-    state.manifest.initial = options.manifest ?? null
-    const contracts = extractContractsFromMeta(state.functions.meta)
-    const baseManifest = state.manifest.initial ?? createEmptyManifest()
-    state.manifest.current = updateManifest(baseManifest, contracts)
-    state.manifest.errors = validateContracts(baseManifest, contracts).errors
-
-    finalizeWorkflows(state)
-    finalizeWorkflowHelperTypes(state)
-    finalizeWorkflowWires(state)
-  }
-
-  // Populate filesAndMethods
-  const startFilesAndMethods = performance.now()
-  const { result, errors } = getFilesAndMethods(state, options.types)
-  state.filesAndMethods = result
-  state.filesAndMethodsErrors = errors
-  logger.debug(
-    `Get files and methods completed in ${(performance.now() - startFilesAndMethods).toFixed(2)}ms`
-  )
-
-  if (!options.setupOnly) {
-    // Everything below re-derives from the state this pass just built, so it
-    // supersedes whatever an earlier pass concluded about a source graph the
-    // run has since finished generating into.
-    logger.beginValidationPass?.()
-    const startAggregate = performance.now()
-    // Apply the inspected auth handler service set before aggregation so it
-    // flows into requiredServices (the generated handler's own func is opaque).
-    stampAuthHandlerServices(state)
-    aggregateRequiredServices(state)
-    logger.debug(
-      `Aggregate required services completed in ${(performance.now() - startAggregate).toFixed(2)}ms`
-    )
-
-    computeResolvedIOTypes(state)
-    computeMiddlewareGroupsMeta(state)
-    computePermissionsGroupsMeta(state)
-    validateNoSecretAliasServices(logger, checker, state)
-    validateSecretUsage(logger, state)
-    computeDiagnostics(state)
-    validateSchemaWiringSeparation(logger, state)
-    validateWorkflowGraphAddons(logger, state)
-    validateScenarioServices(logger, state)
-    validateScenarioSteps(logger, state)
-    // Needs every route, every function and every wireAddon declaration
-    // together, so it can only run once the whole program is inspected.
-    annotateHttpRouteAuth(state)
-    // Last of the validators: they read function meta and every wireAddon
-    // declaration together, so they need the whole program inspected.
-    validateExposedFunctionsGated(logger, state)
-    validateTagsResolveToMiddleware(logger, state)
-
-    if (options.openAPI) {
-      state.openAPISpec = await generateOpenAPISpec(
-        logger,
-        state.functions.meta,
-        state.http.meta,
-        state.schemas,
-        options.openAPI.additionalInfo,
-        pikkuState(null, 'misc', 'errors')
-      )
-    }
-
-    validateAgentModels(logger, state, options.modelAliases)
-    validateAgentToolReferences(logger, state, options)
-    validateAgentScorerReferences(logger, state)
-    validateSecretOverrides(logger, state)
-    validateVariableOverrides(logger, state)
-    validateCredentialOverrides(logger, state)
-    validateRemoteAddonDependencies(logger, state)
-    validateRemoteAddonAuth(logger, state)
-    validateScopeReferences(logger, state)
-    validateSystemRoleScopes(logger, state)
-    validatePersonaRoles(logger, state)
-    logger.endValidationPass?.()
-  }
-
-  const cpu = process.cpuUsage(cpuStart)
-  state.stats = {
-    files: program.getSourceFiles().length,
-    projectFiles: sourceFiles.length,
-    filesReused,
-    types: program.getTypeCount(),
-    instantiations: program.getInstantiationCount(),
-    symbols: program.getSymbolCount(),
-    cpuMs: Math.round((cpu.user + cpu.system) / 1000),
-    wallMs: Math.round(performance.now() - startProgram),
-    heapUsedMb: Math.round(process.memoryUsage().heapUsed / 1048576),
-  }
-  state.program = program
-
-  return state
-}

package/src/schema-generator.ts

@@ -1 +0,0 @@
-export { generateAllSchemas } from './utils/schema-generator.js'

package/src/source-file-cache.ts

@@ -1,73 +0,0 @@
-import * as ts from 'typescript'
-import { createHash } from 'crypto'
-
-/**
- * Parsed source files, kept across programs so a re-inspection only re-parses
- * what changed. Keyed by path; an entry is valid while the file's content
- * hash matches.
- *
- * This is the incremental reuse `oldProgram` used to give, without its cost.
- * Passing the previous ts.Program keeps that program reachable, and a program
- * memoises its type checker — so every re-inspection held a second, fully
- * exercised checker for the sake of skipping a re-parse. A bare SourceFile
- * carries its AST and binder symbols and nothing else; sharing it between
- * programs is what the language service's document registry does.
- *
- * Module-level on purpose: `inspect()` is called several times per `pikku all`
- * (and indefinitely in watch mode) from one process, and the cache is only
- * worth having if it outlives the call.
- */
-const cache = new Map<string, { hash: string; file: ts.SourceFile }>()
-
-export interface SourceFileCacheHost extends ts.CompilerHost {
-  /** Files served from the cache by the program just built. */
-  readonly reused: () => number
-  /**
-   * Drop entries the program did not use, so a file that left the project
-   * (or watch mode's churn) does not pin its AST forever.
-   */
-  readonly prune: (program: ts.Program) => void
-}
-
-export const createSourceFileCacheHost = (
-  options: ts.CompilerOptions
-): SourceFileCacheHost => {
-  const host = ts.createCompilerHost(options, true)
-  let reused = 0
-
-  host.getSourceFile = (fileName, languageVersionOrOptions, onError) => {
-    let text: string | undefined
-    try {
-      text = host.readFile(fileName)
-    } catch (e) {
-      onError?.(e instanceof Error ? e.message : String(e))
-      return undefined
-    }
-    if (text === undefined) return undefined
-
-    const hash = createHash('sha1').update(text).digest('hex')
-    const hit = cache.get(fileName)
-    if (hit && hit.hash === hash) {
-      reused++
-      return hit.file
-    }
-    const file = ts.createSourceFile(
-      fileName,
-      text,
-      languageVersionOrOptions,
-      true
-    )
-    cache.set(fileName, { hash, file })
-    return file
-  }
-
-  return Object.assign(host, {
-    reused: () => reused,
-    prune: (program: ts.Program) => {
-      const live = new Set(program.getSourceFiles().map((sf) => sf.fileName))
-      for (const fileName of cache.keys()) {
-        if (!live.has(fileName)) cache.delete(fileName)
-      }
-    },
-  })
-}