@pikku/cli 0.11.2 → 0.11.3

package/src/utils/pikku-cli-config.ts

@@ -64,6 +64,10 @@ const _getPikkuCLIConfig = async (
           ...extendedConfig.packageMappings,
           ...config.packageMappings,
         },
+        externalPackages: {
+          ...extendedConfig.externalPackages,
+          ...config.externalPackages,
+        },
         ignoreFiles: config.ignoreFiles ??
           extendedConfig.ignoreFiles ?? [
             '**/*.test.ts',
@@ -83,6 +87,7 @@ const _getPikkuCLIConfig = async (
         ...config,
         configDir,
         packageMappings: config.packageMappings || {},
+        externalPackages: config.externalPackages || {},
         rootDir: config.rootDir
           ? resolve(configDir, config.rootDir)
           : configDir,
@@ -134,18 +139,6 @@ const _getPikkuCLIConfig = async (
         'pikku-functions-meta.gen.json'
       )
     }
-    if (!result.functionsMetaMinFile) {
-      result.functionsMetaMinFile = join(
-        functionDir,
-        'pikku-functions-meta.min.gen.ts'
-      )
-    }
-    if (!result.functionsMetaMinJsonFile) {
-      result.functionsMetaMinJsonFile = join(
-        functionDir,
-        'pikku-functions-meta.min.gen.json'
-      )
-    }
     if (!result.functionTypesFile) {
       result.functionTypesFile = join(
         functionDir,
@@ -316,12 +309,6 @@ const _getPikkuCLIConfig = async (
         'pikku-workflow-wirings-meta.gen.ts'
       )
     }
-    if (!result.workflowsWiringMetaJsonFile) {
-      result.workflowsWiringMetaJsonFile = join(
-        workflowDir,
-        'pikku-workflow-wirings-meta.gen.json'
-      )
-    }
     if (!result.workflowsWorkersFile) {
       result.workflowsWorkersFile = join(
         workflowDir,
@@ -341,6 +328,11 @@ const _getPikkuCLIConfig = async (
       )
     }
 
+    // Workflow meta directory (individual JSON files for each workflow)
+    if (!result.workflowMetaDir) {
+      result.workflowMetaDir = join(workflowDir, 'meta')
+    }
+
     // Services
     if (!result.servicesFile) {
       result.servicesFile = join(result.outDir, 'pikku-services.gen.ts')
@@ -374,6 +366,11 @@ const _getPikkuCLIConfig = async (
       result.bootstrapFile = join(result.outDir, 'pikku-bootstrap.gen.ts')
     }
 
+    // Package service factories (for external packages)
+    if (!result.packageFile) {
+      result.packageFile = join(result.outDir, 'pikku-package.gen.ts')
+    }
+
     // MCP
     if (!result.mcpWiringsMetaFile) {
       result.mcpWiringsMetaFile = join(mcpDir, 'pikku-mcp-wirings-meta.gen.ts')
@@ -414,6 +411,18 @@ const _getPikkuCLIConfig = async (
       result.cliTypesFile = join(cliDir, 'pikku-cli-types.gen.ts')
     }
 
+    // Forge
+    const forgeDir = join(result.outDir, 'forge')
+    if (!result.forgeNodesMetaJsonFile) {
+      result.forgeNodesMetaJsonFile = join(
+        forgeDir,
+        'pikku-forge-nodes-meta.gen.json'
+      )
+    }
+    if (!result.forgeTypesFile) {
+      result.forgeTypesFile = join(forgeDir, 'pikku-forge-types.gen.ts')
+    }
+
     if (requiredFields.length > 0) {
       validateCLIConfig(result, requiredFields)
     }
@@ -436,6 +445,30 @@ const _getPikkuCLIConfig = async (
       result.tsconfig = join(result.rootDir, result.tsconfig)
     }
 
+    if (result.externalPackage) {
+      const packageJsonPath = join(result.rootDir, 'package.json')
+      try {
+        const packageJsonContent = await readFile(packageJsonPath, 'utf-8')
+        const packageJson = JSON.parse(packageJsonContent)
+
+        if (
+          !packageJson.name ||
+          typeof packageJson.name !== 'string' ||
+          packageJson.name.trim() === ''
+        ) {
+          throw new Error(
+            `package.json at ${packageJsonPath} is missing a valid "name" field`
+          )
+        }
+
+        result.externalPackageName = packageJson.name
+      } catch (e: any) {
+        throw new Error(
+          `externalPackage is true but could not read or parse package.json at ${packageJsonPath}: ${e.message}`
+        )
+      }
+    }
+
     return result
   } catch (e: any) {
     logger.error(e)

package/src/utils/schema-generator.ts

@@ -3,8 +3,88 @@ import { writeFileInDir } from './file-writer.js'
 import { mkdir, writeFile } from 'fs/promises'
 import { FunctionsMeta, JSONValue } from '@pikku/core'
 import { HTTPWiringsMeta } from '@pikku/core/http'
-import { TypesMap, ErrorCode } from '@pikku/inspector'
+import { TypesMap, ErrorCode, ZodSchemaRef } from '@pikku/inspector'
 import { CLILogger } from '../services/cli-logger.service.js'
+import { tsImport } from 'tsx/esm/api'
+import * as z from 'zod'
+import { zodToTs, createAuxiliaryTypeStore } from 'zod-to-ts'
+import {
+  createPrinter,
+  createSourceFile,
+  EmitHint,
+  ScriptKind,
+  ScriptTarget,
+} from 'typescript'
+
+/**
+ * Convert a name to a valid JavaScript identifier.
+ * Replaces hyphens and other invalid characters with underscores.
+ */
+function toValidIdentifier(name: string): string {
+  let result = name.replace(/[-./]/g, '_')
+  if (/^\d/.test(result)) {
+    result = '_' + result
+  }
+  return result
+}
+
+const PRIMITIVE_TYPES = new Set([
+  'boolean',
+  'string',
+  'number',
+  'null',
+  'undefined',
+  'void',
+  'any',
+  'unknown',
+  'never',
+])
+
+/** Map TypeScript type strings to JSON Schema. Returns null if no schema should be generated. */
+function primitiveTypeToSchema(typeStr: string): JSONValue | null {
+  const normalized = typeStr.trim()
+
+  // void/undefined/never mean no data - don't generate a schema
+  if (
+    normalized === 'void' ||
+    normalized === 'undefined' ||
+    normalized === 'never'
+  ) {
+    return null
+  }
+
+  // Handle boolean literals
+  if (
+    normalized === 'boolean' ||
+    normalized === 'false | true' ||
+    normalized === 'true | false'
+  ) {
+    return { type: 'boolean' }
+  }
+  if (normalized === 'true') {
+    return { const: true }
+  }
+  if (normalized === 'false') {
+    return { const: false }
+  }
+
+  // Handle string
+  if (normalized === 'string') {
+    return { type: 'string' }
+  }
+
+  // Handle number
+  if (normalized === 'number') {
+    return { type: 'number' }
+  }
+
+  // Handle null
+  if (normalized === 'null') {
+    return { type: 'null' }
+  }
+
+  return null
+}
 
 export async function generateSchemas(
   logger: CLILogger,
@@ -13,7 +93,8 @@ export async function generateSchemas(
   functionMeta: FunctionsMeta,
   httpWiringsMeta: HTTPWiringsMeta,
   additionalTypes?: string[],
-  additionalProperties: boolean = false
+  additionalProperties: boolean = false,
+  zodLookup?: Map<string, ZodSchemaRef>
 ): Promise<Record<string, JSONValue>> {
   const schemasSet = new Set(typesMap.customTypes.keys())
   for (const { inputs, outputs } of Object.values(functionMeta)) {
@@ -44,7 +125,6 @@ export async function generateSchemas(
     }
   }
 
-  // Add additional types from schemasFromTypes config
   if (additionalTypes) {
     for (const type of additionalTypes) {
       schemasSet.add(type)
@@ -66,18 +146,28 @@ export async function generateSchemas(
   const schemas: Record<string, JSONValue> = {}
 
   schemasSet.forEach((schema) => {
+    if (PRIMITIVE_TYPES.has(schema)) {
+      return
+    }
+    if (zodLookup?.has(schema)) {
+      return
+    }
     try {
       schemas[schema] = generator.createSchema(schema) as JSONValue
     } catch (e) {
-      // Ignore rootless errors
       if (e instanceof RootlessError) {
-        logger.error(
-          `[${ErrorCode.SCHEMA_NO_ROOT}] Error generating schema since it has no root: ${schema}`
-        )
+        const customType = typesMap.customTypes.get(schema)
+        if (customType) {
+          const primitiveSchema = primitiveTypeToSchema(customType.type)
+          if (primitiveSchema) {
+            schemas[schema] = primitiveSchema
+          }
+        }
         return
       }
+      const customType = typesMap.customTypes.get(schema)
       logger.error(
-        `[${ErrorCode.SCHEMA_GENERATION_ERROR}] Error generating schema: ${schema}. Message: ${e.message}`
+        `[${ErrorCode.SCHEMA_GENERATION_ERROR}] Error generating schema: ${schema}. Message: ${e.message}. Type info: ${customType ? `type=${customType.type}` : 'not in typesMap'}`
       )
     }
   })
@@ -85,6 +175,68 @@ export async function generateSchemas(
   return schemas
 }
 
+export async function generateZodSchemas(
+  logger: CLILogger,
+  zodLookup: Map<string, ZodSchemaRef>,
+  typesMap: TypesMap
+): Promise<Record<string, JSONValue>> {
+  const schemas: Record<string, JSONValue> = {}
+  const auxiliaryTypeStore = createAuxiliaryTypeStore()
+  const printer = createPrinter()
+  const fakeSourceFile = createSourceFile(
+    'zod-types.ts',
+    '',
+    ScriptTarget.ESNext,
+    false,
+    ScriptKind.TS
+  )
+
+  for (const [schemaName, ref] of zodLookup.entries()) {
+    try {
+      const module = await tsImport(ref.sourceFile, import.meta.url)
+      const zodSchema = module[ref.variableName]
+      if (!zodSchema) {
+        logger.warn(
+          `Could not find exported schema '${ref.variableName}' in ${ref.sourceFile} for ${schemaName}. Available exports: ${Object.keys(module).join(', ')}`
+        )
+        continue
+      }
+
+      const schema = z.toJSONSchema(zodSchema) as any
+
+      // Remove fields with defaults from the required array
+      // Fields with defaults are semantically optional in input validation
+      if (schema.required && schema.properties) {
+        schema.required = schema.required.filter((fieldName: string) => {
+          const prop = schema.properties[fieldName]
+          return prop && prop.default === undefined
+        })
+        if (schema.required.length === 0) {
+          delete schema.required
+        }
+      }
+
+      schemas[schemaName] = schema
+      const { node: tsType } = zodToTs(zodSchema, { auxiliaryTypeStore })
+
+      const typeText = printer.printNode(
+        EmitHint.Unspecified,
+        tsType,
+        fakeSourceFile
+      )
+
+      typesMap.addCustomType(schemaName, typeText, [])
+      logger.info(`• Generated schema from Zod: ${schemaName}`)
+    } catch (e) {
+      logger.warn(
+        `Could not convert Zod schema '${schemaName}': ${e instanceof Error ? e.message : e}`
+      )
+    }
+  }
+
+  return schemas
+}
+
 export async function saveSchemas(
   logger: CLILogger,
   schemaParentDir: string,
@@ -92,7 +244,9 @@ export async function saveSchemas(
   typesMap: TypesMap,
   functionsMeta: FunctionsMeta,
   supportsImportAttributes: boolean,
-  additionalTypes?: string[]
+  additionalTypes?: string[],
+  zodLookup?: Map<string, ZodSchemaRef>,
+  packageName?: string | null
 ) {
   await writeFileInDir(
     logger,
@@ -108,26 +262,25 @@ export async function saveSchemas(
           try {
             types.push(typesMap.getUniqueName(inputs[0]))
           } catch {
-            // Skip types not in typesMap (e.g., inline types in generated workflow workers)
+            // Skip types not in typesMap - might be a zod schema name
+            types.push(inputs[0])
           }
         }
         if (outputs?.[0]) {
           try {
             types.push(typesMap.getUniqueName(outputs[0]))
           } catch {
-            // Skip types not in typesMap (e.g., inline types in generated workflow workers)
+            // Skip types not in typesMap - might be a zod schema name
+            types.push(outputs[0])
           }
         }
         return types
       })
       .flat()
-      .filter(
-        (s): s is string =>
-          !!s &&
-          !['boolean', 'string', 'number', 'null', 'undefined'].includes(s)
-      ),
+      .filter((s): s is string => !!s && !PRIMITIVE_TYPES.has(s)),
     ...typesMap.customTypes.keys(),
     ...(additionalTypes || []),
+    ...(zodLookup ? Array.from(zodLookup.keys()) : []),
   ])
 
   if (desiredSchemas.size === 0) {
@@ -153,19 +306,29 @@ export async function saveSchemas(
     (schema) => schemas[schema]
   )
 
+  // Generate the packageName argument for addSchema calls
+  const packageNameArg = packageName ? `, '${packageName}'` : ''
+
   const schemaImports = availableSchemas
-    .map(
-      (schema) => `
-import * as ${schema} from './schemas/${schema}.schema.json' ${supportsImportAttributes ? `with { type: 'json' }` : ''}
-addSchema('${schema}', ${schema})
+    .map((schema) => {
+      const identifier = toValidIdentifier(schema)
+      return `
+import * as ${identifier} from './schemas/${schema}.schema.json' ${supportsImportAttributes ? `with { type: 'json' }` : ''}
+addSchema('${schema}', ${identifier}${packageNameArg})
 `
-    )
+    })
     .join('\n')
 
+  // Only import addSchema if there are schemas to register
+  const importStatement =
+    availableSchemas.length > 0
+      ? `import { addSchema } from '@pikku/core/schema'`
+      : '// No schemas to register'
+
   await writeFileInDir(
     logger,
     `${schemaParentDir}/register.gen.ts`,
-    `import { addSchema } from '@pikku/core/schema'
+    `${importStatement}
 ${schemaImports}`,
     { logWrite: true }
   )

package/src/utils/strip-verbose-meta.ts

@@ -0,0 +1,254 @@
+/**
+ * Utility functions to strip verbose/visualization-only fields from meta objects
+ * for creating minimal runtime-only versions.
+ *
+ * Verbose fields are those used for UI/visualization but not needed at runtime:
+ * - tags, description, summary, errors
+ * - title, displayName
+ * - usedWires, isDirectFunction, services
+ * - stepName (in workflow nodes)
+ *
+ * Note: middleware and permissions are NOT verbose - they're needed at runtime
+ * for resolving tag middleware and permissions in function-runner.ts
+ *
+ * IMPORTANT: We only strip verbose fields at the direct meta entry level.
+ * For most meta types (HTTP, Queue, Scheduler, etc.), entries have pikkuFuncName.
+ * For workflow graphs, we strip at the root level AND from nodes.
+ * We do NOT strip from deeply nested structures like CLI options where 'description' is required.
+ */
+
+const VERBOSE_FIELDS = new Set([
+  'tags',
+  'description',
+  'summary',
+  'errors',
+  'title',
+  'displayName',
+  // Note: middleware and permissions are NOT verbose - they're needed at runtime
+  // for resolving tag middleware and permissions in function-runner.ts
+  'usedWires',
+  'isDirectFunction',
+  'services',
+])
+
+/**
+ * Fields that are verbose only at the node level (workflow nodes)
+ */
+const NODE_VERBOSE_FIELDS = new Set(['stepName'])
+
+/**
+ * Strip verbose fields from an object (shallow - only direct properties)
+ */
+function stripVerboseShallow<T>(obj: T, additionalFields?: Set<string>): T {
+  if (
+    obj === null ||
+    obj === undefined ||
+    typeof obj !== 'object' ||
+    Array.isArray(obj)
+  ) {
+    return obj
+  }
+
+  const result: Record<string, unknown> = {}
+  const fieldsToStrip = additionalFields
+    ? new Set([...VERBOSE_FIELDS, ...additionalFields])
+    : VERBOSE_FIELDS
+
+  for (const [key, value] of Object.entries(obj as Record<string, unknown>)) {
+    if (fieldsToStrip.has(key)) {
+      continue
+    }
+    result[key] = value
+  }
+
+  return result as T
+}
+
+/**
+ * Check if an object has any verbose fields (shallow check)
+ */
+function hasVerboseShallow(
+  obj: unknown,
+  additionalFields?: Set<string>
+): boolean {
+  if (
+    obj === null ||
+    obj === undefined ||
+    typeof obj !== 'object' ||
+    Array.isArray(obj)
+  ) {
+    return false
+  }
+
+  const fieldsToCheck = additionalFields
+    ? new Set([...VERBOSE_FIELDS, ...additionalFields])
+    : VERBOSE_FIELDS
+
+  for (const [key, value] of Object.entries(obj as Record<string, unknown>)) {
+    if (fieldsToCheck.has(key) && value !== undefined) {
+      return true
+    }
+  }
+
+  return false
+}
+
+/**
+ * Strip verbose fields from a meta object.
+ *
+ * Handles different meta structure patterns:
+ * 1. Single workflow graph: { name, pikkuFuncName, nodes: {...}, description, tags }
+ *    -> Strips description, tags at root; strips stepName from each node
+ * 2. Record of meta entries: { "entryName": { pikkuFuncName, description, tags, ... } }
+ *    -> Strips description, tags from each entry (one level deep only)
+ * 3. CLI meta and others with complex nesting: preserved as-is except for pikkuFuncName entries
+ */
+export function stripVerboseFields<T>(obj: T): T {
+  if (obj === null || obj === undefined || typeof obj !== 'object') {
+    return obj
+  }
+
+  if (Array.isArray(obj)) {
+    // Array of entries - strip from each
+    return obj.map((item) => {
+      if (
+        typeof item === 'object' &&
+        item !== null &&
+        'pikkuFuncName' in (item as Record<string, unknown>)
+      ) {
+        return stripVerboseShallow(item)
+      }
+      return item
+    }) as T
+  }
+
+  const objRecord = obj as Record<string, unknown>
+
+  // Case 1: Single workflow graph (has 'nodes' and 'pikkuFuncName' at root)
+  if ('nodes' in objRecord && 'pikkuFuncName' in objRecord) {
+    const result: Record<string, unknown> = {}
+
+    for (const [key, value] of Object.entries(objRecord)) {
+      if (VERBOSE_FIELDS.has(key)) {
+        continue
+      }
+
+      if (key === 'nodes' && typeof value === 'object' && value !== null) {
+        // Strip stepName from each node
+        const nodes: Record<string, unknown> = {}
+        for (const [nodeId, node] of Object.entries(
+          value as Record<string, unknown>
+        )) {
+          nodes[nodeId] = stripVerboseShallow(node, NODE_VERBOSE_FIELDS)
+        }
+        result[key] = nodes
+      } else {
+        result[key] = value
+      }
+    }
+
+    return result as T
+  }
+
+  // Case 2: Record of meta entries (each entry has pikkuFuncName)
+  // Only strip at exactly one level deep - the immediate children
+  const result: Record<string, unknown> = {}
+
+  for (const [key, value] of Object.entries(objRecord)) {
+    if (typeof value === 'object' && value !== null && !Array.isArray(value)) {
+      const valueObj = value as Record<string, unknown>
+      // Only strip from entries that look like meta entries (have pikkuFuncName)
+      if ('pikkuFuncName' in valueObj) {
+        result[key] = stripVerboseShallow(value)
+      } else {
+        // Keep nested structures as-is (e.g., CLI programs, options)
+        result[key] = value
+      }
+    } else {
+      result[key] = value
+    }
+  }
+
+  return result as T
+}
+
+/**
+ * Check if an object has any verbose fields that would be stripped
+ */
+export function hasVerboseFields(obj: unknown): boolean {
+  if (obj === null || obj === undefined || typeof obj !== 'object') {
+    return false
+  }
+
+  if (Array.isArray(obj)) {
+    return obj.some((item) => {
+      if (
+        typeof item === 'object' &&
+        item !== null &&
+        'pikkuFuncName' in (item as Record<string, unknown>)
+      ) {
+        return hasVerboseShallow(item)
+      }
+      return false
+    })
+  }
+
+  const objRecord = obj as Record<string, unknown>
+
+  // Case 1: Single workflow graph
+  if ('nodes' in objRecord && 'pikkuFuncName' in objRecord) {
+    // Check root level
+    if (hasVerboseShallow(objRecord)) {
+      return true
+    }
+    // Check nodes for stepName
+    const nodes = objRecord.nodes
+    if (typeof nodes === 'object' && nodes !== null) {
+      for (const node of Object.values(nodes as Record<string, unknown>)) {
+        if (hasVerboseShallow(node, NODE_VERBOSE_FIELDS)) {
+          return true
+        }
+      }
+    }
+    return false
+  }
+
+  // Case 2: Record of meta entries
+  for (const value of Object.values(objRecord)) {
+    if (typeof value === 'object' && value !== null && !Array.isArray(value)) {
+      const valueObj = value as Record<string, unknown>
+      if ('pikkuFuncName' in valueObj && hasVerboseShallow(value)) {
+        return true
+      }
+    }
+  }
+
+  return false
+}
+
+/**
+ * Write both minimal and verbose meta files
+ * Only writes verbose file if it differs from minimal
+ *
+ * @returns Object with paths that were written
+ */
+export async function writeMetaFiles(
+  writeFile: (path: string, content: string) => Promise<void>,
+  basePath: string,
+  meta: unknown
+): Promise<{ minimal: string; verbose?: string }> {
+  const minimalMeta = stripVerboseFields(meta)
+  const minimalPath = basePath.replace(/\.gen\.json$/, '.gen.json')
+  const verbosePath = basePath.replace(/\.gen\.json$/, '-verbose.gen.json')
+
+  // Always write minimal
+  await writeFile(minimalPath, JSON.stringify(minimalMeta, null, 2))
+
+  // Only write verbose if it has additional fields
+  if (hasVerboseFields(meta)) {
+    await writeFile(verbosePath, JSON.stringify(meta, null, 2))
+    return { minimal: minimalPath, verbose: verbosePath }
+  }
+
+  return { minimal: minimalPath }
+}