@pikku/cli 0.7.7 → 0.8.1

package/src/events/queue/serialize-queue-map.ts

@@ -0,0 +1,119 @@
+import type { queueWorkersMeta } from '@pikku/core/queue'
+import { serializeImportMap } from '../../serialize-import-map.js'
+import { TypesMap } from '@pikku/inspector'
+import { FunctionsMeta } from '@pikku/core'
+import { generateCustomTypes } from '../../utils.js'
+
+export const serializeQueueMap = (
+  relativeToPath: string,
+  packageMappings: Record<string, string>,
+  typesMap: TypesMap,
+  functionsMeta: FunctionsMeta,
+  queueWorkersMeta: queueWorkersMeta
+) => {
+  const requiredTypes = new Set<string>()
+  const serializedCustomTypes = generateCustomTypes(typesMap, requiredTypes)
+  const serializedQueues = generateQueues(
+    queueWorkersMeta,
+    functionsMeta,
+    typesMap,
+    requiredTypes
+  )
+
+  const serializedImportMap = serializeImportMap(
+    relativeToPath,
+    packageMappings,
+    typesMap,
+    requiredTypes
+  )
+
+  return `/**
+ * This provides the structure needed for typescript to be aware of Queue workers and their input/output types
+ */
+    
+${serializedImportMap}
+${serializedCustomTypes}
+
+import type { QueueJob } from '@pikku/core/queue'
+
+interface QueueHandler<I, O> {
+    input: I;
+    output: O;
+}
+
+${serializedQueues}
+
+type QueueAdd = <Name extends keyof QueueMap>(
+  name: Name,
+  data: QueueMap[Name]['input'],
+  options?: {
+    priority?: number
+    delay?: number
+    attempts?: number
+    removeOnComplete?: number
+    removeOnFail?: number
+    jobId?: string
+  }
+) => Promise<string>
+
+type QueueGetJob = <Name extends keyof QueueMap>(
+  name: Name,
+  jobId: string
+) => Promise<QueueJob<QueueMap[Name]['input'], QueueMap[Name]['output']> | null>
+
+export type TypedPikkuQueue = {
+  add: QueueAdd;
+  getJob: QueueGetJob;
+}
+  `
+}
+
+function generateQueues(
+  queueWorkersMeta: queueWorkersMeta,
+  functionsMeta: FunctionsMeta,
+  typesMap: TypesMap,
+  requiredTypes: Set<string>
+) {
+  // Initialize an object to collect queues
+  const queuesObj: Record<string, { inputType: string; outputType: string }> =
+    {}
+
+  // Iterate through Queue metadata
+  for (const [queueName, { pikkuFuncName }] of Object.entries(
+    queueWorkersMeta
+  )) {
+    const functionMeta = functionsMeta[pikkuFuncName]
+    if (!functionMeta) {
+      throw new Error(
+        `Function ${queueName} not found in functionsMeta. Please check your configuration.`
+      )
+    }
+
+    const input = functionMeta.inputs ? functionMeta.inputs[0] : undefined
+    const output = functionMeta.outputs ? functionMeta.outputs[0] : undefined
+
+    // Store the input and output types for QueueHandler
+    const inputType = input ? typesMap.getTypeMeta(input).uniqueName : 'null'
+    const outputType = output ? typesMap.getTypeMeta(output).uniqueName : 'null'
+
+    requiredTypes.add(inputType)
+    requiredTypes.add(outputType)
+
+    // Add Queue entry
+    queuesObj[queueName] = {
+      inputType,
+      outputType,
+    }
+  }
+
+  // Build the Queues object as a string
+  let queuesStr = 'export type QueueMap = {\n'
+
+  for (const [queueName, handler] of Object.entries(queuesObj)) {
+    queuesStr += `  readonly '${queueName}': QueueHandler<${handler.inputType}, ${handler.outputType}>,\n`
+  }
+
+  queuesStr += '};\n'
+
+  return queuesStr
+}

package/src/events/queue/serialize-queue-meta.ts

@@ -0,0 +1,10 @@
+import { queueWorkersMeta } from '@pikku/core'
+
+export const serializeQueueMeta = (queueWorkersMeta: queueWorkersMeta) => {
+  const serializedOutput: string[] = []
+  serializedOutput.push("import { pikkuState } from '@pikku/core'")
+  serializedOutput.push(
+    `pikkuState('queue', 'meta', ${JSON.stringify(queueWorkersMeta, null, 2)})`
+  )
+  return serializedOutput.join('\n')
+}

package/src/events/queue/serialize-queue-wrapper.ts

@@ -0,0 +1,35 @@
+export const serializeQueueWrapper = (queueMapPath: string) => {
+  return `
+import type { QueueService, QueueJob } from '@pikku/core/queue'
+import type { QueueMap, TypedPikkuQueue } from '${queueMapPath}'
+
+export class PikkuQueue implements TypedPikkuQueue {
+    constructor(private queueService: QueueService) {}
+
+    public async add<Name extends keyof QueueMap>(
+        queueName: Name,
+        data: QueueMap[Name]['input'],
+        options?: {
+            priority?: number
+            delay?: number
+            attempts?: number
+            removeOnComplete?: number
+            removeOnFail?: number
+            jobId?: string
+        }
+    ): Promise<string> {
+        return this.queueService.add(queueName as string, data, options);
+    }
+
+    public async getJob<Name extends keyof QueueMap>(
+        queueName: Name,
+        jobId: string
+    ): Promise<QueueJob<QueueMap[Name]['input'], QueueMap[Name]['output']> | null> {
+        return this.queueService.getJob<QueueMap[Name]['input'], QueueMap[Name]['output']>(
+            queueName as string,
+            jobId
+        );
+    }
+}
+`
+}

package/src/events/rpc/pikku-command-rpc-client.ts

@@ -0,0 +1,33 @@
+import { serializeRPCWrapper } from './serialize-rpc-wrapper.js'
+import {
+  getFileImportRelativePath,
+  logCommandInfoAndTime,
+  writeFileInDir,
+} from '../../utils.js'
+import { PikkuCommandWithoutState } from '../../types.js'
+
+export const pikkuRPCClient: PikkuCommandWithoutState = async (
+  logger,
+  { rpcFile, rpcMapDeclarationFile, packageMappings }
+) => {
+  return await logCommandInfoAndTime(
+    logger,
+    'Generating RPC wrapper',
+    'Generated RPC wrapper',
+    [rpcFile === undefined, "rpcFile isn't set in the pikku config"],
+    async () => {
+      if (!rpcFile) {
+        throw new Error("rpcFile isn't set in the pikku config")
+      }
+
+      const rpcMapDeclarationPath = getFileImportRelativePath(
+        rpcFile,
+        rpcMapDeclarationFile,
+        packageMappings
+      )
+
+      const content = [serializeRPCWrapper(rpcMapDeclarationPath)]
+      await writeFileInDir(logger, rpcFile, content.join('\n'))
+    }
+  )
+}

package/src/events/rpc/pikku-command-rpc-map.ts

@@ -0,0 +1,26 @@
+import { logCommandInfoAndTime, writeFileInDir } from '../../utils.js'
+import { serializeTypedRPCMap } from './serialize-typed-rpc-map.js'
+import { PikkuCommand } from '../../types.js'
+
+export const pikkuRPCMap: PikkuCommand = async (
+  logger,
+  { rpcMapDeclarationFile, packageMappings },
+  { functions, rpc }
+) => {
+  return await logCommandInfoAndTime(
+    logger,
+    'Creating RPC map',
+    'Created RPC map',
+    [false],
+    async () => {
+      const content = serializeTypedRPCMap(
+        rpcMapDeclarationFile,
+        packageMappings,
+        functions.typesMap,
+        functions.meta,
+        rpc.meta
+      )
+      await writeFileInDir(logger, rpcMapDeclarationFile, content)
+    }
+  )
+}

package/src/events/rpc/pikku-command-rpc.ts

@@ -0,0 +1,22 @@
+import { logCommandInfoAndTime, writeFileInDir } from '../../utils.js'
+import { PikkuCommand } from '../../types.js'
+
+export const pikkuRPC: PikkuCommand = async (
+  logger,
+  { rpcMetaFile },
+  { rpc }
+) => {
+  return await logCommandInfoAndTime(
+    logger,
+    'Finding RPCs tasks',
+    'Found RPCs',
+    [false],
+    async () => {
+      await writeFileInDir(
+        logger,
+        rpcMetaFile,
+        `import { pikkuState } from '@pikku/core'\npikkuState('rpc', 'meta', ${JSON.stringify(rpc.meta, null, 2)})`
+      )
+    }
+  )
+}

package/{bin → src/events/rpc}/pikku-rpc.ts

@@ -1,17 +1,19 @@
-import { PikkuCLIConfig } from '../src/pikku-cli-config.js'
-import { InspectorState } from '@pikku/inspector'
-import { logCommandInfoAndTime, writeFileInDir } from '../src/utils/utils.js'
+import { logCommandInfoAndTime, writeFileInDir } from '../../utils.js'
+import { PikkuCommand } from '../../types.js'
 
-export const pikkuRPC = async (
-  { rpcMetaFile }: PikkuCLIConfig,
-  { rpc }: InspectorState
+export const pikkuRPC: PikkuCommand = async (
+  logger,
+  { rpcMetaFile },
+  { rpc }
 ) => {
   return await logCommandInfoAndTime(
+    logger,
     'Finding RPCs tasks',
     'Found RPCs',
     [false],
     async () => {
       await writeFileInDir(
+        logger,
         rpcMetaFile,
         `import { pikkuState } from '@pikku/core'\npikkuState('rpc', 'meta', ${JSON.stringify(rpc.meta, null, 2)})`
       )

package/src/events/rpc/serialize-rpc-wrapper.ts

@@ -0,0 +1,29 @@
+export const serializeRPCWrapper = (rpcMapPath: string) => {
+  return `
+import { PikkuFetch } from "./pikku-fetch.gen.js"
+import type { RPCInvoke } from '${rpcMapPath}'
+
+export class PikkuRPC {
+    pikkuFetch = new PikkuFetch()
+
+    setPikkuFetch(pikkuFetch: PikkuFetch): void {
+        this.pikkuFetch = pikkuFetch
+    }
+
+    setServerUrl(serverUrl: string): void {
+        this.pikkuFetch.setServerUrl(serverUrl)
+    }
+
+    setAuthorizationJWT(jwt: string | null): void {
+        this.pikkuFetch.setAuthorizationJWT(jwt)
+    }
+
+    // Generic RPC invoke method
+    invoke: RPCInvoke = async (name, data) => {
+       return await this.pikkuFetch.post('/rpc', { name, data })
+    }
+}
+
+export const pikkuRPC = new PikkuRPC();
+`
+}

package/src/{serialize-typed-rpc-map.ts → events/rpc/serialize-typed-rpc-map.ts}

@@ -1,8 +1,9 @@
-import { serializeImportMap } from './utils/serialize-import-map.js'
+import type { RPCMeta } from '@pikku/core/rpc'
+
+import { serializeImportMap } from '../../serialize-import-map.js'
 import { TypesMap } from '@pikku/inspector'
 import { FunctionsMeta } from '@pikku/core'
-import { generateCustomTypes } from './utils/utils.js'
-import { RPCMeta } from '../../core/src/rpc/rpc-types.js'
+import { generateCustomTypes } from '../../utils.js'
 
 export const serializeTypedRPCMap = (
   relativeToPath: string,
@@ -41,7 +42,7 @@ interface RPCHandler<I, O> {
 
 ${serializedRPCs}
 
-type RPCInvoke = <Name extends keyof RPCMap>(
+export type RPCInvoke = <Name extends keyof RPCMap>(
   name: Name,
   data: RPCMap[Name]['input'],
   options?: {

package/{bin/pikku-scheduler.ts → src/events/scheduler/pikku-command-scheduler.ts}

@@ -1,17 +1,18 @@
-import { PikkuCLIConfig } from '../src/pikku-cli-config.js'
-import { InspectorState } from '@pikku/inspector'
 import {
   logCommandInfoAndTime,
   serializeFileImports,
   writeFileInDir,
-} from '../src/utils/utils.js'
-import { serializeSchedulerMeta } from '../src/serialize-scheduler-meta.js'
+} from '../../utils.js'
+import { serializeSchedulerMeta } from './serialize-scheduler-meta.js'
+import { PikkuCommand } from '../../types.js'
 
-export const pikkuScheduler = async (
-  cliConfig: PikkuCLIConfig,
-  visitState: InspectorState
+export const pikkuScheduler: PikkuCommand = async (
+  logger,
+  cliConfig,
+  visitState
 ) => {
   return await logCommandInfoAndTime(
+    logger,
     'Finding scheduled tasks',
     'Found scheduled tasks',
     [visitState.scheduledTasks.files.size === 0],
@@ -19,13 +20,15 @@ export const pikkuScheduler = async (
       const { schedulersFile, schedulersMetaFile, packageMappings } = cliConfig
       const { scheduledTasks } = visitState
       await writeFileInDir(
+        logger,
         schedulersMetaFile,
         serializeSchedulerMeta(scheduledTasks.meta)
       )
       await writeFileInDir(
+        logger,
         schedulersFile,
         serializeFileImports(
-          'addSerializedTasks',
+          'addScheduledTasks',
           schedulersFile,
           scheduledTasks.files,
           packageMappings

package/src/inspector-glob.ts

@@ -1,15 +1,17 @@
 import * as path from 'path'
 import { glob } from 'tinyglobby'
 import { InspectorFilters, InspectorState, inspect } from '@pikku/inspector'
-import { logCommandInfoAndTime } from './utils/utils.js'
+import { logCommandInfoAndTime } from './utils.js'
 
 export const inspectorGlob = async (
+  logger,
   rootDir: string,
   srcDirectories: string[],
   filters: InspectorFilters
 ) => {
   let result: InspectorState
   await logCommandInfoAndTime(
+    logger,
     'Inspecting codebase',
     'Inspected codebase',
     [false],
@@ -21,7 +23,7 @@ export const inspectorGlob = async (
           )
         )
       ).flat()
-      result = await inspect(routeFiles, filters)
+      result = await inspect(logger, routeFiles, filters)
     }
   )
   return result!

package/src/pikku-cli-config.ts

@@ -1,6 +1,6 @@
 import { join, dirname, resolve, isAbsolute } from 'path'
 import { readdir, readFile } from 'fs/promises'
-import { OpenAPISpecInfo } from './openapi-spec-generator.js'
+import { OpenAPISpecInfo } from './events/http/openapi-spec-generator.js'
 import { InspectorFilters } from '@pikku/inspector'
 import { PikkuEventTypes } from '@pikku/core'
 
@@ -34,6 +34,18 @@ export interface PikkuCLICoreOutputFiles {
   schedulersFile: string
   schedulersMetaFile: string
 
+  // Queue processors
+  queueWorkersFile: string
+  queueWorkersMetaFile: string
+  queueMapDeclarationFile: string
+
+  // MCP
+  mcpEndpointsFile: string
+  mcpEndpointsMetaFile: string
+
+  // Services
+  servicesFile: string
+
   // Application bootstrap
   bootstrapFile: string
   bootstrapFiles: Record<PikkuEventTypes, string>
@@ -56,12 +68,17 @@ export type PikkuCLIConfig = {
   nextHTTPFile?: string
   fetchFile?: string
   websocketFile?: string
+  rpcFile?: string
+  queueFile?: string
+  mcpJsonFile?: string
 
   openAPI?: {
     outputFile: string
     additionalInfo: OpenAPISpecInfo
   }
 
+  middlewareServices?: string[]
+
   filters: InspectorFilters
 } & PikkuCLICoreOutputFiles
 
@@ -70,18 +87,21 @@ const CONFIG_DIR_FILES = [
   'nextHTTPFile',
   'fetchFile',
   'websocketFile',
+  'rpcFile',
+  'queueFile',
+  'mcpJsonFile',
 ]
 
 export const getPikkuCLIConfig = async (
   configFile: string | undefined = undefined,
   requiredFields: Array<keyof PikkuCLIConfig>,
-  tags: string[] = [],
+  filters: InspectorFilters = {},
   exitProcess: boolean = false
 ): Promise<PikkuCLIConfig> => {
   const config = await _getPikkuCLIConfig(
     configFile,
     requiredFields,
-    tags,
+    filters,
     exitProcess
   )
   return config
@@ -90,7 +110,7 @@ export const getPikkuCLIConfig = async (
 const _getPikkuCLIConfig = async (
   configFile: string | undefined = undefined,
   requiredFields: Array<keyof PikkuCLIConfig>,
-  tags: string[] = [],
+  filters: InspectorFilters = {},
   exitProcess: boolean = false
 ): Promise<PikkuCLIConfig> => {
   if (!configFile) {
@@ -118,7 +138,7 @@ const _getPikkuCLIConfig = async (
       const extendedConfig = await getPikkuCLIConfig(
         resolve(configDir, config.extends),
         [],
-        tags,
+        filters,
         exitProcess
       )
       result = {
@@ -142,77 +162,128 @@ const _getPikkuCLIConfig = async (
     }
 
     if (result.outDir) {
+      // Create transport/event directories
+      const functionDir = join(result.outDir, 'function')
+      const httpDir = join(result.outDir, 'http')
+      const channelDir = join(result.outDir, 'channel')
+      const rpcDir = join(result.outDir, 'rpc')
+      const schedulerDir = join(result.outDir, 'scheduler')
+      const queueDir = join(result.outDir, 'queue')
+      const mcpDir = join(result.outDir, 'mcp')
+
+      // Create directories if they don't exist (will be done lazily when files are written)
+
       if (!result.schemaDirectory) {
-        result.schemaDirectory = join(result.outDir, 'pikku-schemas')
+        result.schemaDirectory = join(result.outDir, 'schemas')
       }
+
+      // Functions
       if (!result.functionsFile) {
-        result.functionsFile = join(result.outDir, 'pikku-functions.gen.ts')
+        result.functionsFile = join(functionDir, 'pikku-functions.gen.ts')
       }
       if (!result.functionsMetaFile) {
         result.functionsMetaFile = join(
-          result.outDir,
+          functionDir,
           'pikku-functions-meta.gen.ts'
         )
       }
-      if (!result.rpcMetaFile) {
-        result.rpcMetaFile = join(result.outDir, 'pikku-rpc-meta.gen.ts')
-      }
-      if (!result.rpcMapDeclarationFile) {
-        result.rpcMapDeclarationFile = join(
-          result.outDir,
-          'pikku-rpc-map.gen.ts'
-        )
+      if (!result.typesDeclarationFile) {
+        result.typesDeclarationFile = join(result.outDir, 'pikku-types.gen.ts')
       }
+
+      // HTTP
       if (!result.httpRoutesFile) {
-        result.httpRoutesFile = join(result.outDir, 'pikku-http-routes.gen.ts')
+        result.httpRoutesFile = join(httpDir, 'pikku-http-routes.gen.ts')
       }
       if (!result.httpRoutesMetaFile) {
         result.httpRoutesMetaFile = join(
-          result.outDir,
+          httpDir,
           'pikku-http-routes-meta.gen.ts'
         )
       }
-      if (!result.schedulersFile) {
-        result.schedulersFile = join(result.outDir, 'pikku-schedules.gen.ts')
-      }
-      if (!result.schedulersMetaFile) {
-        result.schedulersMetaFile = join(
-          result.outDir,
-          'pikku-schedules-meta.gen.ts'
+      if (!result.httpRoutesMapDeclarationFile) {
+        result.httpRoutesMapDeclarationFile = join(
+          httpDir,
+          'pikku-http-routes-map.gen.d.ts'
         )
       }
+
+      // Channels/WebSocket
       if (!result.channelsFile) {
-        result.channelsFile = join(result.outDir, 'pikku-channels.gen.ts')
+        result.channelsFile = join(channelDir, 'pikku-channels.gen.ts')
       }
       if (!result.channelsMetaFile) {
-        result.channelsMetaFile = join(
-          result.outDir,
-          'pikku-channels-meta.gen.ts'
+        result.channelsMetaFile = join(channelDir, 'pikku-channels-meta.gen.ts')
+      }
+      if (!result.channelsMapDeclarationFile) {
+        result.channelsMapDeclarationFile = join(
+          channelDir,
+          'pikku-channels-map.gen.d.ts'
         )
       }
-      if (!result.typesDeclarationFile) {
-        result.typesDeclarationFile = join(result.outDir, 'pikku-types.gen.ts')
+
+      // RPC
+      if (!result.rpcMetaFile) {
+        result.rpcMetaFile = join(rpcDir, 'pikku-rpc-meta.gen.ts')
       }
-      if (!result.httpRoutesMapDeclarationFile) {
-        result.httpRoutesMapDeclarationFile = join(
-          result.outDir,
-          'pikku-http-routes-map.gen.d.ts'
+      if (!result.rpcMapDeclarationFile) {
+        result.rpcMapDeclarationFile = join(rpcDir, 'pikku-rpc-map.gen.ts')
+      }
+
+      // Scheduler
+      if (!result.schedulersFile) {
+        result.schedulersFile = join(schedulerDir, 'pikku-scheduler.gen.ts')
+      }
+      if (!result.schedulersMetaFile) {
+        result.schedulersMetaFile = join(
+          schedulerDir,
+          'pikku-scheduler-meta.gen.ts'
         )
       }
-      if (!result.channelsMapDeclarationFile) {
-        result.channelsMapDeclarationFile = join(
-          result.outDir,
-          'pikku-channels-map.gen.d.ts'
+
+      // Queue
+      if (!result.queueWorkersFile) {
+        result.queueWorkersFile = join(queueDir, 'pikku-queue-workers.gen.ts')
+      }
+      if (!result.queueWorkersMetaFile) {
+        result.queueWorkersMetaFile = join(
+          queueDir,
+          'pikku-queue-workers-meta.gen.ts'
         )
       }
+      if (!result.queueMapDeclarationFile) {
+        result.queueMapDeclarationFile = join(
+          queueDir,
+          'pikku-queue-map.gen.ts'
+        )
+      }
+
+      // Services
+      if (!result.servicesFile) {
+        result.servicesFile = join(result.outDir, 'pikku-services.gen.ts')
+      }
+
+      // Bootstrap files
       if (!result.bootstrapFile) {
         result.bootstrapFile = join(result.outDir, 'pikku-bootstrap.gen.ts')
       }
 
+      // MCP
+      if (!result.mcpEndpointsMetaFile) {
+        result.mcpEndpointsMetaFile = join(mcpDir, 'mcp-endpoints-meta.gen.ts')
+      }
+      if (!result.mcpEndpointsFile) {
+        result.mcpEndpointsFile = join(mcpDir, 'mcp-endpoints.gen.ts')
+      }
+      if (!result.mcpJsonFile) {
+        result.mcpJsonFile = join(mcpDir, 'mcp.gen.json')
+      }
+
       result.bootstrapFiles = result.bootstrapFiles || {}
       for (const key of Object.keys(PikkuEventTypes)) {
+        const eventDir = join(result.outDir, key.toLowerCase())
         result.bootstrapFiles[key] = join(
-          result.outDir,
+          eventDir,
           `pikku-bootstrap-${key}.gen.ts`
         )
       }
@@ -236,8 +307,14 @@ const _getPikkuCLIConfig = async (
     }
 
     result.filters = result.filters || {}
-    if (tags.length > 0) {
-      result.filters.tags = tags
+    if (filters.tags && filters.tags.length > 0) {
+      result.filters.tags = filters.tags
+    }
+    if (filters.types && filters.types.length > 0) {
+      result.filters.types = filters.types
+    }
+    if (filters.directories && filters.directories.length > 0) {
+      result.filters.directories = filters.directories
     }
 
     if (!isAbsolute(result.tsconfig)) {