@pikku/cli 0.7.7 → 0.8.0

package/src/{serialize-typed-http-map.ts → events/http/serialize-typed-http-map.ts}

@@ -1,5 +1,5 @@
 import { HTTPRoutesMeta } from '@pikku/core/http'
-import { serializeImportMap } from './utils/serialize-import-map.js'
+import { serializeImportMap } from '../../serialize-import-map.js'
 import { MetaInputTypes, TypesMap } from '@pikku/inspector'
 import { FunctionsMeta } from '@pikku/core'
 

package/src/events/mcp/pikku-command-mcp-json.ts

@@ -0,0 +1,33 @@
+import { logCommandInfoAndTime, writeFileInDir } from '../../utils.js'
+import { serializeMCPJson } from './serialize-mcp-json.js'
+import { PikkuCommand } from '../../types.js'
+
+export const pikkuMCPJSON: PikkuCommand = async (
+  logger,
+  { mcpJsonFile, schemaDirectory },
+  { mcpEndpoints, functions }
+) => {
+  return await logCommandInfoAndTime(
+    logger,
+    'Generating MCP JSON',
+    'Generated MCP JSON',
+    [mcpEndpoints.files.size === 0 || !mcpJsonFile],
+    async () => {
+      // Generate MCP JSON file
+      if (mcpJsonFile) {
+        const mcpJson = await serializeMCPJson(
+          logger,
+          schemaDirectory,
+          functions.meta,
+          functions.typesMap,
+          mcpEndpoints.resourcesMeta,
+          mcpEndpoints.toolsMeta,
+          mcpEndpoints.promptsMeta
+        )
+        await writeFileInDir(logger, mcpJsonFile, mcpJson, {
+          ignoreModifyComment: true,
+        })
+      }
+    }
+  )
+}

package/src/events/mcp/pikku-command-mcp.ts

@@ -0,0 +1,110 @@
+import {
+  logCommandInfoAndTime,
+  serializeFileImports,
+  writeFileInDir,
+} from '../../utils.js'
+import { PikkuCommand } from '../../types.js'
+import { readFile } from 'fs/promises'
+import { join } from 'path'
+
+// Helper function to generate arguments from schema
+const generateArgumentsFromSchema = async (
+  inputSchema: string | null,
+  schemaDirectory: string,
+  typesMap: any,
+  logger: any
+): Promise<Array<{ name: string; description: string; required: boolean }>> => {
+  if (!inputSchema) return []
+
+  if (
+    ['boolean', 'string', 'number', 'null', 'undefined', 'void'].includes(
+      inputSchema
+    )
+  ) {
+    return []
+  }
+
+  const uniqueName = typesMap.getUniqueName(inputSchema)
+  if (!uniqueName) return []
+
+  try {
+    const schemaPath = join(
+      schemaDirectory,
+      'schemas',
+      `${uniqueName}.schema.json`
+    )
+    const schemaContent = await readFile(schemaPath, 'utf-8')
+    const schema = JSON.parse(schemaContent)
+
+    const argumentsArray: Array<{
+      name: string
+      description: string
+      required: boolean
+    }> = []
+    if (schema && typeof schema === 'object' && schema.properties) {
+      const properties = schema.properties as Record<string, any>
+      const required = (schema.required as string[]) || []
+
+      for (const [propName, propSchema] of Object.entries(properties)) {
+        argumentsArray.push({
+          name: propName,
+          description: propSchema.description || `${propName} parameter`,
+          required: required.includes(propName),
+        })
+      }
+    }
+    return argumentsArray
+  } catch (e) {
+    logger.warn(`Could not load schema for type: ${uniqueName}`)
+    return []
+  }
+}
+
+export const pikkuMCP: PikkuCommand = async (
+  logger,
+  { mcpEndpointsFile, mcpEndpointsMetaFile, packageMappings, schemaDirectory },
+  { mcpEndpoints, functions }
+) => {
+  return await logCommandInfoAndTime(
+    logger,
+    'Finding MCP endpoints',
+    'Found MCP endpoints',
+    [mcpEndpoints.files.size === 0],
+    async () => {
+      await writeFileInDir(
+        logger,
+        mcpEndpointsFile,
+        serializeFileImports(
+          'addMCPResource or addMCPTool',
+          mcpEndpointsFile,
+          mcpEndpoints.files,
+          packageMappings
+        )
+      )
+
+      // Populate arguments for prompts meta before serializing
+      const promptsMetaWithArguments = { ...mcpEndpoints.promptsMeta }
+      for (const promptMeta of Object.values(promptsMetaWithArguments)) {
+        const functionMeta = functions.meta[promptMeta.pikkuFuncName]
+        if (functionMeta) {
+          const inputType = functionMeta.inputs?.[0]
+          promptMeta.arguments = await generateArgumentsFromSchema(
+            inputType || null,
+            schemaDirectory || '',
+            functions.typesMap,
+            logger
+          )
+        }
+      }
+
+      await writeFileInDir(
+        logger,
+        mcpEndpointsMetaFile,
+        `import { pikkuState } from '@pikku/core'
+pikkuState('mcp', 'resourcesMeta', ${JSON.stringify(mcpEndpoints.resourcesMeta, null, 2)})
+pikkuState('mcp', 'toolsMeta', ${JSON.stringify(mcpEndpoints.toolsMeta, null, 2)})
+pikkuState('mcp', 'promptsMeta', ${JSON.stringify(promptsMetaWithArguments, null, 2)})`
+      )
+    }
+  )
+}

package/src/events/mcp/serialize-mcp-json.ts

@@ -0,0 +1,159 @@
+import { readFile } from 'fs/promises'
+import { join } from 'path'
+import { MCPResourceMeta, MCPToolMeta, MCPPromptMeta } from '@pikku/core'
+import { FunctionsMeta, JSONValue } from '@pikku/core'
+import { TypesMap } from '@pikku/inspector'
+import { CLILogger } from '../../utils.js'
+
+interface MCPEndpoint {
+  name: string
+  description?: string
+  parameters?: JSONValue
+  returns?: JSONValue
+  streaming?: boolean
+}
+
+export const serializeMCPJson = async (
+  logger: CLILogger,
+  schemaDirectory: string,
+  functionsMeta: FunctionsMeta,
+  typesMap: TypesMap,
+  mcpResourceMeta: MCPResourceMeta,
+  mcpToolMeta: MCPToolMeta,
+  mcpPromptMeta: MCPPromptMeta
+): Promise<string> => {
+  const tools: MCPEndpoint[] = []
+  const resources: MCPEndpoint[] = []
+  const prompts: any[] = []
+
+  // Helper function to load schema from file
+  const loadSchema = async (
+    typeName: string | undefined
+  ): Promise<JSONValue | undefined> => {
+    if (
+      !typeName ||
+      ['boolean', 'string', 'number', 'null', 'undefined', 'void'].includes(
+        typeName
+      )
+    ) {
+      return undefined
+    }
+
+    const uniqueName = typesMap.getUniqueName(typeName)
+    if (!uniqueName) {
+      return undefined
+    }
+
+    try {
+      const schemaPath = join(
+        schemaDirectory,
+        'schemas',
+        `${uniqueName}.schema.json`
+      )
+      const schemaContent = await readFile(schemaPath, 'utf-8')
+      return JSON.parse(schemaContent)
+    } catch (e) {
+      logger.warn(`Could not load schema for type: ${uniqueName}`)
+      return undefined
+    }
+  }
+
+  // Process MCP resources
+  for (const [name, endpointMeta] of Object.entries(mcpResourceMeta)) {
+    const functionMeta = functionsMeta[endpointMeta.pikkuFuncName]
+    if (!functionMeta) {
+      logger.warn(
+        `Function ${endpointMeta.pikkuFuncName} not found in functionsMeta. Skipping resource ${name}.`
+      )
+      continue
+    }
+
+    const inputType = functionMeta.inputs?.[0]
+    const outputType = functionMeta.outputs?.[0]
+
+    const parameters = await loadSchema(inputType)
+    const returns = await loadSchema(outputType)
+
+    const endpoint = {
+      uri: name,
+      name,
+      description: endpointMeta.description,
+      ...(parameters && { parameters }),
+      ...(returns && { returns }),
+      ...(endpointMeta.streaming && { streaming: true }),
+    }
+
+    resources.push(endpoint)
+  }
+
+  // Process MCP tools
+  for (const [name, endpointMeta] of Object.entries(mcpToolMeta)) {
+    const functionMeta = functionsMeta[endpointMeta.pikkuFuncName]
+    if (!functionMeta) {
+      logger.warn(
+        `Function ${endpointMeta.pikkuFuncName} not found in functionsMeta. Skipping tool ${name}.`
+      )
+      continue
+    }
+
+    const inputType = functionMeta.inputs?.[0]
+    const outputType = functionMeta.outputs?.[0]
+
+    const parameters = await loadSchema(inputType)
+    const returns = await loadSchema(outputType)
+
+    const endpoint = {
+      name,
+      description: endpointMeta.description,
+      ...(parameters && { parameters }),
+      ...(returns && { returns }),
+      ...(endpointMeta.streaming && { streaming: true }),
+    }
+
+    tools.push(endpoint)
+  }
+
+  // Process MCP prompts
+  for (const [name, endpointMeta] of Object.entries(mcpPromptMeta)) {
+    const functionMeta = functionsMeta[endpointMeta.pikkuFuncName]
+    if (!functionMeta) {
+      logger.warn(
+        `Function ${endpointMeta.pikkuFuncName} not found in functionsMeta. Skipping prompt ${name}.`
+      )
+      continue
+    }
+
+    const inputType = functionMeta.inputs?.[0]
+    // TODO: this needs to be a json schema type, not any
+    const inputSchema: any = await loadSchema(inputType)
+
+    // Generate arguments from input schema
+    const argumentsArray: any[] = []
+    if (
+      inputSchema &&
+      typeof inputSchema === 'object' &&
+      inputSchema.properties
+    ) {
+      const properties = inputSchema.properties as Record<string, any>
+      const required = (inputSchema.required as string[]) || []
+
+      for (const [propName, propSchema] of Object.entries(properties)) {
+        argumentsArray.push({
+          name: propName,
+          description: propSchema.description || `${propName} parameter`,
+          required: required.includes(propName),
+        })
+      }
+    }
+
+    const prompt = {
+      name,
+      description: endpointMeta.description,
+      arguments: argumentsArray,
+    }
+
+    prompts.push(prompt)
+  }
+
+  return JSON.stringify({ tools, resources, prompts }, null, 2)
+}

package/src/events/queue/pikku-command-queue-map.ts

@@ -0,0 +1,26 @@
+import { logCommandInfoAndTime, writeFileInDir } from '../../utils.js'
+import { serializeQueueMap } from './serialize-queue-map.js'
+import { PikkuCommand } from '../../types.js'
+
+export const pikkuQueueMap: PikkuCommand = async (
+  logger,
+  { queueMapDeclarationFile, packageMappings },
+  { queueWorkers, functions }
+) => {
+  return await logCommandInfoAndTime(
+    logger,
+    'Creating Queue map',
+    'Created Queue map',
+    [queueWorkers.files.size === 0],
+    async () => {
+      const content = serializeQueueMap(
+        queueMapDeclarationFile,
+        packageMappings,
+        functions.typesMap,
+        functions.meta,
+        queueWorkers.meta
+      )
+      await writeFileInDir(logger, queueMapDeclarationFile, content)
+    }
+  )
+}

package/src/events/queue/pikku-command-queue-service.ts

@@ -0,0 +1,33 @@
+import { serializeQueueWrapper } from './serialize-queue-wrapper.js'
+import {
+  getFileImportRelativePath,
+  logCommandInfoAndTime,
+  writeFileInDir,
+} from '../../utils.js'
+import { PikkuCommandWithoutState } from '../../types.js'
+
+export const pikkuQueueService: PikkuCommandWithoutState = async (
+  logger,
+  { queueFile, queueMapDeclarationFile, packageMappings }
+) => {
+  return await logCommandInfoAndTime(
+    logger,
+    'Generating queue service wrapper',
+    'Generated queue service wrapper',
+    [queueFile === undefined, "queueFile isn't set in the pikku config"],
+    async () => {
+      if (!queueFile) {
+        throw new Error("queueFile is isn't set in the pikku config")
+      }
+
+      const queueMapDeclarationPath = getFileImportRelativePath(
+        queueFile,
+        queueMapDeclarationFile,
+        packageMappings
+      )
+
+      const content = [serializeQueueWrapper(queueMapDeclarationPath)]
+      await writeFileInDir(logger, queueFile, content.join('\n'))
+    }
+  )
+}

package/src/events/queue/pikku-command-queue.ts

@@ -0,0 +1,42 @@
+import { PikkuCLIConfig } from '../../pikku-cli-config.js'
+import { InspectorState } from '@pikku/inspector'
+import {
+  logCommandInfoAndTime,
+  serializeFileImports,
+  writeFileInDir,
+} from '../../utils.js'
+import { serializeQueueMeta } from './serialize-queue-meta.js'
+import { PikkuCommand } from '../../types.js'
+
+export const pikkuQueue: PikkuCommand = async (
+  logger,
+  cliConfig: PikkuCLIConfig,
+  visitState: InspectorState
+) => {
+  return await logCommandInfoAndTime(
+    logger,
+    'Finding queues',
+    'Found queue',
+    [visitState.queueWorkers.files.size === 0],
+    async () => {
+      const { queueWorkersFile, queueWorkersMetaFile, packageMappings } =
+        cliConfig
+      const { queueWorkers } = visitState
+      await writeFileInDir(
+        logger,
+        queueWorkersMetaFile,
+        serializeQueueMeta(queueWorkers.meta)
+      )
+      await writeFileInDir(
+        logger,
+        queueWorkersFile,
+        serializeFileImports(
+          'addQueueWorkers',
+          queueWorkersFile,
+          queueWorkers.files,
+          packageMappings
+        )
+      )
+    }
+  )
+}

package/src/events/queue/pikku-queue-map.ts

@@ -0,0 +1,26 @@
+import { logCommandInfoAndTime, writeFileInDir } from '../../utils.js'
+import { serializeQueueMap } from './serialize-queue-map.js'
+import { PikkuCommand } from '../../types.js'
+
+export const pikkuQueueMap: PikkuCommand = async (
+  logger,
+  { queueMapDeclarationFile, packageMappings },
+  { queueWorkers, functions }
+) => {
+  return await logCommandInfoAndTime(
+    logger,
+    'Creating Queue map',
+    'Created Queue map',
+    [queueWorkers.files.size === 0],
+    async () => {
+      const content = serializeQueueMap(
+        queueMapDeclarationFile,
+        packageMappings,
+        functions.typesMap,
+        functions.meta,
+        queueWorkers.meta
+      )
+      await writeFileInDir(logger, queueMapDeclarationFile, content)
+    }
+  )
+}

package/src/events/queue/pikku-queue.ts

@@ -0,0 +1,40 @@
+import {
+  logCommandInfoAndTime,
+  serializeFileImports,
+  writeFileInDir,
+} from '../../utils.js'
+import { serializeQueueMeta } from './serialize-queue-meta.js'
+import { PikkuCommand } from '../../types.js'
+
+export const pikkuQueue: PikkuCommand = async (
+  logger,
+  cliConfig,
+  visitState
+) => {
+  return await logCommandInfoAndTime(
+    logger,
+    'Finding queues',
+    'Found queue',
+    [visitState.queueWorkers.files.size === 0],
+    async () => {
+      const { queueWorkersFile, queueWorkersMetaFile, packageMappings } =
+        cliConfig
+      const { queueWorkers } = visitState
+      await writeFileInDir(
+        logger,
+        queueWorkersMetaFile,
+        serializeQueueMeta(queueWorkers.meta)
+      )
+      await writeFileInDir(
+        logger,
+        queueWorkersFile,
+        serializeFileImports(
+          'addQueueWorkers',
+          queueWorkersFile,
+          queueWorkers.files,
+          packageMappings
+        )
+      )
+    }
+  )
+}

package/src/events/queue/serialize-queue-map.ts

@@ -0,0 +1,119 @@
+import type { queueWorkersMeta } from '@pikku/core/queue'
+import { serializeImportMap } from '../../serialize-import-map.js'
+import { TypesMap } from '@pikku/inspector'
+import { FunctionsMeta } from '@pikku/core'
+import { generateCustomTypes } from '../../utils.js'
+
+export const serializeQueueMap = (
+  relativeToPath: string,
+  packageMappings: Record<string, string>,
+  typesMap: TypesMap,
+  functionsMeta: FunctionsMeta,
+  queueWorkersMeta: queueWorkersMeta
+) => {
+  const requiredTypes = new Set<string>()
+  const serializedCustomTypes = generateCustomTypes(typesMap, requiredTypes)
+  const serializedQueues = generateQueues(
+    queueWorkersMeta,
+    functionsMeta,
+    typesMap,
+    requiredTypes
+  )
+
+  const serializedImportMap = serializeImportMap(
+    relativeToPath,
+    packageMappings,
+    typesMap,
+    requiredTypes
+  )
+
+  return `/**
+ * This provides the structure needed for typescript to be aware of Queue workers and their input/output types
+ */
+    
+${serializedImportMap}
+${serializedCustomTypes}
+
+import type { QueueJob } from '@pikku/core/queue'
+
+interface QueueHandler<I, O> {
+    input: I;
+    output: O;
+}
+
+${serializedQueues}
+
+type QueueAdd = <Name extends keyof QueueMap>(
+  name: Name,
+  data: QueueMap[Name]['input'],
+  options?: {
+    priority?: number
+    delay?: number
+    attempts?: number
+    removeOnComplete?: number
+    removeOnFail?: number
+    jobId?: string
+  }
+) => Promise<string>
+
+type QueueGetJob = <Name extends keyof QueueMap>(
+  name: Name,
+  jobId: string
+) => Promise<QueueJob<QueueMap[Name]['input'], QueueMap[Name]['output']> | null>
+
+export type TypedPikkuQueue = {
+  add: QueueAdd;
+  getJob: QueueGetJob;
+}
+  `
+}
+
+function generateQueues(
+  queueWorkersMeta: queueWorkersMeta,
+  functionsMeta: FunctionsMeta,
+  typesMap: TypesMap,
+  requiredTypes: Set<string>
+) {
+  // Initialize an object to collect queues
+  const queuesObj: Record<string, { inputType: string; outputType: string }> =
+    {}
+
+  // Iterate through Queue metadata
+  for (const [queueName, { pikkuFuncName }] of Object.entries(
+    queueWorkersMeta
+  )) {
+    const functionMeta = functionsMeta[pikkuFuncName]
+    if (!functionMeta) {
+      throw new Error(
+        `Function ${queueName} not found in functionsMeta. Please check your configuration.`
+      )
+    }
+
+    const input = functionMeta.inputs ? functionMeta.inputs[0] : undefined
+    const output = functionMeta.outputs ? functionMeta.outputs[0] : undefined
+
+    // Store the input and output types for QueueHandler
+    const inputType = input ? typesMap.getTypeMeta(input).uniqueName : 'null'
+    const outputType = output ? typesMap.getTypeMeta(output).uniqueName : 'null'
+
+    requiredTypes.add(inputType)
+    requiredTypes.add(outputType)
+
+    // Add Queue entry
+    queuesObj[queueName] = {
+      inputType,
+      outputType,
+    }
+  }
+
+  // Build the Queues object as a string
+  let queuesStr = 'export type QueueMap = {\n'
+
+  for (const [queueName, handler] of Object.entries(queuesObj)) {
+    queuesStr += `  readonly '${queueName}': QueueHandler<${handler.inputType}, ${handler.outputType}>,\n`
+  }
+
+  queuesStr += '};\n'
+
+  return queuesStr
+}

package/src/events/queue/serialize-queue-meta.ts

@@ -0,0 +1,10 @@
+import { queueWorkersMeta } from '@pikku/core'
+
+export const serializeQueueMeta = (queueWorkersMeta: queueWorkersMeta) => {
+  const serializedOutput: string[] = []
+  serializedOutput.push("import { pikkuState } from '@pikku/core'")
+  serializedOutput.push(
+    `pikkuState('queue', 'meta', ${JSON.stringify(queueWorkersMeta, null, 2)})`
+  )
+  return serializedOutput.join('\n')
+}

package/src/events/queue/serialize-queue-wrapper.ts

@@ -0,0 +1,35 @@
+export const serializeQueueWrapper = (queueMapPath: string) => {
+  return `
+import type { QueueService, QueueJob } from '@pikku/core/queue'
+import type { QueueMap, TypedPikkuQueue } from '${queueMapPath}'
+
+export class PikkuQueue implements TypedPikkuQueue {
+    constructor(private queueService: QueueService) {}
+
+    public async add<Name extends keyof QueueMap>(
+        queueName: Name,
+        data: QueueMap[Name]['input'],
+        options?: {
+            priority?: number
+            delay?: number
+            attempts?: number
+            removeOnComplete?: number
+            removeOnFail?: number
+            jobId?: string
+        }
+    ): Promise<string> {
+        return this.queueService.add(queueName as string, data, options);
+    }
+
+    public async getJob<Name extends keyof QueueMap>(
+        queueName: Name,
+        jobId: string
+    ): Promise<QueueJob<QueueMap[Name]['input'], QueueMap[Name]['output']> | null> {
+        return this.queueService.getJob<QueueMap[Name]['input'], QueueMap[Name]['output']>(
+            queueName as string,
+            jobId
+        );
+    }
+}
+`
+}

package/src/events/rpc/pikku-command-rpc-map.ts

@@ -0,0 +1,26 @@
+import { logCommandInfoAndTime, writeFileInDir } from '../../utils.js'
+import { serializeTypedRPCMap } from './serialize-typed-rpc-map.js'
+import { PikkuCommand } from '../../types.js'
+
+export const pikkuRPCMap: PikkuCommand = async (
+  logger,
+  { rpcMapDeclarationFile, packageMappings },
+  { functions, rpc }
+) => {
+  return await logCommandInfoAndTime(
+    logger,
+    'Creating RPC map',
+    'Created RPC map',
+    [false],
+    async () => {
+      const content = serializeTypedRPCMap(
+        rpcMapDeclarationFile,
+        packageMappings,
+        functions.typesMap,
+        functions.meta,
+        rpc.meta
+      )
+      await writeFileInDir(logger, rpcMapDeclarationFile, content)
+    }
+  )
+}

package/src/events/rpc/pikku-command-rpc.ts

@@ -0,0 +1,22 @@
+import { logCommandInfoAndTime, writeFileInDir } from '../../utils.js'
+import { PikkuCommand } from '../../types.js'
+
+export const pikkuRPC: PikkuCommand = async (
+  logger,
+  { rpcMetaFile },
+  { rpc }
+) => {
+  return await logCommandInfoAndTime(
+    logger,
+    'Finding RPCs tasks',
+    'Found RPCs',
+    [false],
+    async () => {
+      await writeFileInDir(
+        logger,
+        rpcMetaFile,
+        `import { pikkuState } from '@pikku/core'\npikkuState('rpc', 'meta', ${JSON.stringify(rpc.meta, null, 2)})`
+      )
+    }
+  )
+}