@pikku/core 0.11.1 → 0.12.0

package/src/types/state.types.ts

@@ -0,0 +1,169 @@
+import { PikkuError, ErrorDetails } from '../errors/error-handler.js'
+import {
+  CorePikkuFunctionConfig,
+  CorePermissionGroup,
+  CorePikkuPermission,
+} from '../function/functions.types.js'
+import { CorePikkuTriggerFunctionConfig } from '../wirings/trigger/trigger.types.js'
+import { CoreChannel, ChannelsMeta } from '../wirings/channel/channel.types.js'
+import { CLIMeta, CLIProgramState } from '../wirings/cli/cli.types.js'
+import {
+  HTTPMethod,
+  CoreHTTPFunctionWiring,
+  HTTPWiringsMeta,
+} from '../wirings/http/http.types.js'
+import {
+  CoreMCPResource,
+  MCPResourceMeta,
+  MCPToolMeta,
+  CoreMCPPrompt,
+  MCPPromptMeta,
+} from '../wirings/mcp/mcp.types.js'
+import { CoreAIAgent, AIAgentMeta } from '../wirings/ai-agent/ai-agent.types.js'
+import {
+  CoreQueueWorker,
+  QueueWorkersMeta,
+} from '../wirings/queue/queue.types.js'
+import {
+  CoreScheduledTask,
+  ScheduledTasksMeta,
+} from '../wirings/scheduler/scheduler.types.js'
+import {
+  CoreWorkflow,
+  WorkflowsRuntimeMeta,
+} from '../wirings/workflow/workflow.types.js'
+import {
+  CoreTrigger,
+  CoreTriggerSource,
+  TriggerMeta,
+  TriggerSourceMeta,
+} from '../wirings/trigger/trigger.types.js'
+import {
+  FunctionsMeta,
+  CorePikkuMiddleware,
+  CorePikkuMiddlewareGroup,
+  CreateConfig,
+  CreateSingletonServices,
+  CreateWireServices,
+  CoreConfig,
+  CoreSingletonServices,
+  CoreServices,
+  CoreUserSession,
+} from './core.types.js'
+import type { CorePikkuChannelMiddleware } from '../wirings/channel/channel.types.js'
+
+/**
+ * State structure for an individual package
+ */
+export interface PikkuPackageState {
+  function: {
+    meta: FunctionsMeta
+    functions: Map<string, CorePikkuFunctionConfig<any, any>>
+  }
+  rpc: {
+    meta: Record<string, string>
+    files: Map<
+      string,
+      {
+        exportedName: string
+        path: string
+      }
+    >
+    /** Maps namespace aliases to package config (e.g., 'ext' -> { package: '@pikku/...', rpcEndpoint: '...' }) */
+    externalPackages: Map<string, { package: string; rpcEndpoint?: string }>
+  }
+  http: {
+    middleware: Map<string, CorePikkuMiddleware<any, any>[]>
+    permissions: Map<string, CorePermissionGroup | CorePikkuPermission[]>
+    routes: Map<HTTPMethod, Map<string, CoreHTTPFunctionWiring<any, any, any>>>
+    meta: HTTPWiringsMeta
+  }
+  channel: {
+    channels: Map<string, CoreChannel<any, any>>
+    meta: ChannelsMeta
+  }
+  scheduler: {
+    tasks: Map<string, CoreScheduledTask>
+    meta: ScheduledTasksMeta
+  }
+  queue: {
+    registrations: Map<string, CoreQueueWorker>
+    meta: QueueWorkersMeta
+  }
+  workflows: {
+    registrations: Map<string, CoreWorkflow>
+    meta: WorkflowsRuntimeMeta
+  }
+  trigger: {
+    functions: Map<string, CorePikkuTriggerFunctionConfig<any, any>>
+    triggers: Map<string, CoreTrigger>
+    triggerSources: Map<string, CoreTriggerSource>
+    meta: TriggerMeta
+    sourceMeta: TriggerSourceMeta
+  }
+  mcp: {
+    resources: Map<string, CoreMCPResource>
+    resourcesMeta: MCPResourceMeta
+    toolsMeta: MCPToolMeta
+    prompts: Map<string, CoreMCPPrompt>
+    promptsMeta: MCPPromptMeta
+  }
+  agent: {
+    agents: Map<string, CoreAIAgent>
+    agentsMeta: AIAgentMeta
+  }
+  cli: {
+    meta: CLIMeta | Record<string, any> // Backward compatible with old published CLI format
+    programs: Record<string, CLIProgramState>
+  }
+  middleware: {
+    tagGroup: Record<string, CorePikkuMiddlewareGroup>
+    httpGroup: Record<string, CorePikkuMiddlewareGroup>
+  }
+  channelMiddleware: {
+    tagGroup: Record<string, CorePikkuChannelMiddleware[]>
+  }
+  permissions: {
+    tagGroup: Record<string, CorePermissionGroup | CorePikkuPermission[]>
+    httpGroup: Record<string, CorePermissionGroup | CorePikkuPermission[]>
+  }
+  misc: {
+    errors: Map<PikkuError, ErrorDetails>
+    schemas: Map<string, any>
+    middleware: Record<string, CorePikkuMiddleware[]>
+    channelMiddleware: Record<string, CorePikkuChannelMiddleware[]>
+    permissions: Record<string, CorePermissionGroup | CorePikkuPermission[]>
+  }
+  models: {
+    config: {
+      models?: Record<
+        string,
+        string | { model: string; temperature?: number; maxSteps?: number }
+      >
+      agentDefaults?: { temperature?: number; maxSteps?: number }
+      agentOverrides?: Record<
+        string,
+        { model?: string; temperature?: number; maxSteps?: number }
+      >
+    } | null
+  }
+  package: {
+    /** Service factory functions for external packages */
+    factories: {
+      createConfig?: CreateConfig<CoreConfig>
+      createSingletonServices?: CreateSingletonServices<
+        CoreConfig,
+        CoreSingletonServices
+      >
+      createWireServices?: CreateWireServices<
+        CoreSingletonServices,
+        CoreServices,
+        CoreUserSession
+      >
+    } | null
+    /** Cached singleton services for this package */
+    singletonServices: CoreSingletonServices | null
+    /** Absolute path to this package's .pikku directory */
+    metaDir: string | null
+  }
+}

package/src/utils.ts

@@ -1,14 +1,12 @@
 import { Logger } from './services/logger.js'
+import { CoreSingletonServices, WireServices } from './types/core.types.js'
 
-// TODO: WireServices probably needs it's own type
-// but is an issue for the future and will be tackled
-// with dependency injection
 export const closeWireServices = async (
   logger: Logger,
-  wireServices: Record<string, any>
+  wireServices: WireServices
 ) => {
   await Promise.all(
-    Object.values(wireServices).map(async (service) => {
+    Object.values(wireServices).map(async (service: any) => {
       if (service?.close) {
         try {
           await service.close()
@@ -39,6 +37,23 @@ export const isSerializable = (data: any): boolean => {
   )
 }
 
+export const getTagGroups = <T>(
+  tagGroups: Record<string, T>,
+  tag: string
+): T[] => {
+  const results: T[] = []
+  const exact = tagGroups[tag]
+  if (exact) results.push(exact)
+  let colonIdx = tag.lastIndexOf(':')
+  while (colonIdx !== -1) {
+    const parent = tag.slice(0, colonIdx)
+    const group = tagGroups[parent]
+    if (group) results.push(group)
+    colonIdx = parent.lastIndexOf(':')
+  }
+  return results
+}
+
 const EMPTY_ARRAY = Object.freeze([])
 
 export const freezeDedupe = <T>(
@@ -57,3 +72,57 @@ export const freezeDedupe = <T>(
   }
   return Object.freeze(out)
 }
+
+/**
+ * Stop a single service by calling its stop method if it exists
+ */
+const stopService = async (
+  logger: Logger,
+  name: string,
+  service: any
+): Promise<void> => {
+  const stop = service?.stop
+  if (stop) {
+    logger.info(`Stopping singleton service: ${name}`)
+    try {
+      await stop.call(service)
+    } catch (e: any) {
+      logger.error(`Error stopping service ${name}:`, e)
+    }
+  }
+}
+
+/**
+ * Stop all singleton services, including external package services.
+ * External package services are stopped first, then the parent services.
+ *
+ * @param singletonServices - The parent singleton services to stop
+ */
+export const stopSingletonServices = async (
+  singletonServices: CoreSingletonServices
+): Promise<void> => {
+  const logger = singletonServices.logger
+
+  // First, stop all external package singleton services
+  if (globalThis.pikkuState) {
+    for (const [packageName, packageState] of globalThis.pikkuState) {
+      // Skip main package - we handle it separately
+      if (packageName === '__main__') continue
+
+      const packageServices = packageState.package?.singletonServices
+      if (packageServices) {
+        logger.info(`Stopping singleton services for package: ${packageName}`)
+        for (const [name, service] of Object.entries(packageServices)) {
+          await stopService(logger, `${packageName}/${name}`, service)
+        }
+        // Clear the cached services
+        packageState.package.singletonServices = null
+      }
+    }
+  }
+
+  // Then stop the parent singleton services
+  for (const [name, service] of Object.entries(singletonServices)) {
+    await stopService(logger, name, service)
+  }
+}

package/src/version.test.ts

@@ -0,0 +1,80 @@
+import { strict as assert } from 'assert'
+import { describe, test } from 'node:test'
+import {
+  formatVersionedId,
+  parseVersionedId,
+  isVersionedId,
+} from './version.js'
+
+describe('formatVersionedId', () => {
+  test('formats base name with version', () => {
+    assert.strictEqual(formatVersionedId('createUser', 1), 'createUser@v1')
+    assert.strictEqual(formatVersionedId('createUser', 2), 'createUser@v2')
+    assert.strictEqual(formatVersionedId('createUser', 10), 'createUser@v10')
+  })
+
+  test('handles namespaced base names', () => {
+    assert.strictEqual(
+      formatVersionedId('pkg:createUser', 3),
+      'pkg:createUser@v3'
+    )
+  })
+})
+
+describe('parseVersionedId', () => {
+  test('parses versioned IDs', () => {
+    assert.deepStrictEqual(parseVersionedId('createUser@v1'), {
+      baseName: 'createUser',
+      version: 1,
+    })
+    assert.deepStrictEqual(parseVersionedId('createUser@v2'), {
+      baseName: 'createUser',
+      version: 2,
+    })
+    assert.deepStrictEqual(parseVersionedId('createUser@v10'), {
+      baseName: 'createUser',
+      version: 10,
+    })
+  })
+
+  test('returns null version for unversioned IDs', () => {
+    assert.deepStrictEqual(parseVersionedId('createUser'), {
+      baseName: 'createUser',
+      version: null,
+    })
+  })
+
+  test('returns null version for invalid version suffix', () => {
+    assert.deepStrictEqual(parseVersionedId('createUser@vx'), {
+      baseName: 'createUser@vx',
+      version: null,
+    })
+    assert.deepStrictEqual(parseVersionedId('createUser@v0'), {
+      baseName: 'createUser@v0',
+      version: null,
+    })
+    assert.deepStrictEqual(parseVersionedId('createUser@v-1'), {
+      baseName: 'createUser@v-1',
+      version: null,
+    })
+  })
+
+  test('handles namespaced versioned IDs', () => {
+    assert.deepStrictEqual(parseVersionedId('pkg:createUser@v3'), {
+      baseName: 'pkg:createUser',
+      version: 3,
+    })
+  })
+})
+
+describe('isVersionedId', () => {
+  test('returns true for versioned IDs', () => {
+    assert.strictEqual(isVersionedId('createUser@v1'), true)
+    assert.strictEqual(isVersionedId('createUser@v99'), true)
+  })
+
+  test('returns false for unversioned IDs', () => {
+    assert.strictEqual(isVersionedId('createUser'), false)
+    assert.strictEqual(isVersionedId('createUser@vx'), false)
+  })
+})

package/src/version.ts

@@ -0,0 +1,25 @@
+const VERSION_SEPARATOR = '@v'
+
+export function formatVersionedId(baseName: string, version: number): string {
+  return `${baseName}${VERSION_SEPARATOR}${version}`
+}
+
+export function parseVersionedId(id: string): {
+  baseName: string
+  version: number | null
+} {
+  const idx = id.lastIndexOf(VERSION_SEPARATOR)
+  if (idx === -1) {
+    return { baseName: id, version: null }
+  }
+  const versionStr = id.slice(idx + VERSION_SEPARATOR.length)
+  const version = Number(versionStr)
+  if (!Number.isInteger(version) || version < 1) {
+    return { baseName: id, version: null }
+  }
+  return { baseName: id.slice(0, idx), version }
+}
+
+export function isVersionedId(id: string): boolean {
+  return parseVersionedId(id).version !== null
+}

package/src/wirings/ai-agent/ai-agent-memory.ts

@@ -0,0 +1,300 @@
+import { randomUUID } from 'crypto'
+import type { CoreSingletonServices } from '../../types/core.types.js'
+import type {
+  CoreAIAgent,
+  AIAgentMemoryConfig,
+  AIAgentInput,
+  AIMessage,
+} from './ai-agent.types.js'
+import type { AIStorageService } from '../../services/ai-storage-service.js'
+import type { Logger } from '../../services/logger.js'
+import type { SchemaService } from '../../services/schema-service.js'
+
+export function resolveMemoryServices(
+  agent: CoreAIAgent,
+  singletonServices: CoreSingletonServices
+): {
+  storage: AIStorageService | undefined
+} {
+  const memoryConfig = agent.memory
+  const storage = memoryConfig?.storage
+    ? (singletonServices as any)[memoryConfig.storage]
+    : singletonServices.aiStorage
+  return { storage }
+}
+
+export function deepMergeWorkingMemory(
+  existing: Record<string, unknown>,
+  updates: Record<string, unknown>
+): Record<string, unknown> {
+  const result = { ...existing }
+  for (const key of Object.keys(updates)) {
+    const value = updates[key]
+    if (value === null) {
+      delete result[key]
+    } else if (
+      typeof value === 'object' &&
+      !Array.isArray(value) &&
+      typeof result[key] === 'object' &&
+      result[key] !== null &&
+      !Array.isArray(result[key])
+    ) {
+      result[key] = deepMergeWorkingMemory(
+        result[key] as Record<string, unknown>,
+        value as Record<string, unknown>
+      )
+    } else {
+      result[key] = value
+    }
+  }
+  return result
+}
+
+export function buildWorkingMemoryPrompt(
+  currentState: Record<string, unknown> | null,
+  jsonSchema?: Record<string, unknown>
+): string {
+  const parts: string[] = []
+
+  if (jsonSchema?.properties) {
+    const props = jsonSchema.properties as Record<
+      string,
+      { type?: string; description?: string }
+    >
+    const fieldLines = Object.entries(props).map(([name, def]) => {
+      const type = def.type ?? 'unknown'
+      const desc = def.description ? ` - ${def.description}` : ''
+      return `  - ${name} (${type})${desc}`
+    })
+    if (fieldLines.length > 0) {
+      parts.push(`Working memory fields:\n${fieldLines.join('\n')}`)
+    }
+  }
+
+  if (currentState && Object.keys(currentState).length > 0) {
+    parts.push(`Current working memory:\n${JSON.stringify(currentState)}`)
+  } else {
+    parts.push('Current working memory: (empty)')
+  }
+
+  parts.push(
+    'When you learn new information, output a partial JSON update in <working_memory> tags. ' +
+      'Only include changed fields. Set a field to null to delete it.'
+  )
+
+  return parts.join('\n\n')
+}
+
+export async function loadContextMessages(
+  memoryConfig: AIAgentMemoryConfig | undefined,
+  storage: AIStorageService | undefined,
+  input: AIAgentInput,
+  workingMemoryJsonSchema?: Record<string, unknown>
+): Promise<AIMessage[]> {
+  const contextMessages: AIMessage[] = []
+
+  if (memoryConfig?.workingMemory && storage) {
+    const workingMem = await storage.getWorkingMemory(input.threadId, 'thread')
+    const prompt = buildWorkingMemoryPrompt(workingMem, workingMemoryJsonSchema)
+    contextMessages.push({
+      id: `wm-${randomUUID()}`,
+      role: 'system',
+      content: prompt,
+      createdAt: new Date(),
+    })
+  }
+
+  return contextMessages
+}
+
+export async function saveMessages(
+  storage: AIStorageService | undefined,
+  threadId: string,
+  resourceId: string,
+  memoryConfig: AIAgentMemoryConfig | undefined,
+  userMessage: AIMessage,
+  result: {
+    text: string
+    steps: {
+      toolCalls?: {
+        name: string
+        args: Record<string, unknown>
+        result: string
+      }[]
+    }[]
+  },
+  options?: {
+    workingMemoryJsonSchema?: Record<string, unknown>
+    workingMemorySchemaName?: string | null
+    logger?: Logger
+    schemaService?: SchemaService
+  }
+): Promise<string> {
+  let responseText = result.text
+
+  if (storage) {
+    const newMessages: AIMessage[] = [userMessage]
+
+    for (const step of result.steps) {
+      if (step.toolCalls?.length) {
+        newMessages.push({
+          id: randomUUID(),
+          role: 'assistant',
+          toolCalls: step.toolCalls.map((tc) => ({
+            id: randomUUID(),
+            name: tc.name,
+            args: tc.args,
+          })),
+          createdAt: new Date(),
+        })
+        newMessages.push({
+          id: randomUUID(),
+          role: 'tool',
+          toolResults: step.toolCalls.map((tc) => ({
+            id: randomUUID(),
+            name: tc.name,
+            result: tc.result,
+          })),
+          createdAt: new Date(),
+        })
+      }
+    }
+
+    newMessages.push({
+      id: randomUUID(),
+      role: 'assistant',
+      content: responseText,
+      createdAt: new Date(),
+    })
+
+    await storage.saveMessages(threadId, newMessages)
+
+    if (memoryConfig?.workingMemory) {
+      const parsed = parseWorkingMemory(responseText)
+      if (parsed) {
+        const existing =
+          (await storage.getWorkingMemory(threadId, 'thread')) ?? {}
+        const merged = deepMergeWorkingMemory(existing, parsed)
+
+        if (options?.schemaService && options?.workingMemorySchemaName) {
+          try {
+            await options.schemaService.validateSchema(
+              options.workingMemorySchemaName,
+              merged
+            )
+          } catch (err) {
+            options.logger?.warn(
+              `Working memory validation failed: ${err instanceof Error ? err.message : String(err)}`
+            )
+          }
+        }
+
+        await storage.saveWorkingMemory(threadId, 'thread', merged)
+        responseText = stripWorkingMemoryTags(responseText)
+      }
+    }
+  }
+
+  return responseText
+}
+
+export function trimMessages(
+  messages: AIMessage[],
+  maxTokenBudget: number = 100000
+): AIMessage[] {
+  const sanitized = sanitizeToolMessages(messages)
+
+  let estimatedTokens = 0
+  const result: AIMessage[] = []
+
+  for (let i = sanitized.length - 1; i >= 0; i--) {
+    const msg = sanitized[i]
+    const msgTokens = estimateTokens(msg)
+    if (estimatedTokens + msgTokens > maxTokenBudget && result.length > 0) {
+      break
+    }
+    estimatedTokens += msgTokens
+    result.unshift(msg)
+  }
+
+  if (
+    result.length > 0 &&
+    result[0].role !== 'user' &&
+    result[0].role !== 'system'
+  ) {
+    const firstUserIdx = result.findIndex(
+      (m) => m.role === 'user' || m.role === 'system'
+    )
+    if (firstUserIdx > 0) {
+      return result.slice(firstUserIdx)
+    }
+  }
+
+  return result
+}
+
+function sanitizeToolMessages(messages: AIMessage[]): AIMessage[] {
+  const result: AIMessage[] = []
+
+  for (let i = 0; i < messages.length; i++) {
+    const msg = messages[i]
+
+    if (msg.role === 'assistant' && msg.toolCalls?.length) {
+      const next = messages[i + 1]
+      if (!next || next.role !== 'tool' || !next.toolResults?.length) {
+        if (msg.content) {
+          result.push({ ...msg, toolCalls: undefined })
+        }
+        continue
+      }
+
+      const toolCallIds = new Set(msg.toolCalls.map((tc) => tc.id))
+      const resultIds = new Set(next.toolResults.map((tr) => tr.id))
+      const allMatched = [...toolCallIds].every((id) => resultIds.has(id))
+
+      if (!allMatched) {
+        if (msg.content) {
+          result.push({ ...msg, toolCalls: undefined })
+        }
+        result.push({
+          ...next,
+          toolResults:
+            next.toolResults.filter((tr) => !toolCallIds.has(tr.id)) ||
+            undefined,
+        })
+        i++
+        continue
+      }
+    }
+
+    result.push(msg)
+  }
+
+  return result.filter(
+    (m) => m.role !== 'tool' || (m.toolResults && m.toolResults.length > 0)
+  )
+}
+
+function estimateTokens(msg: AIMessage): number {
+  let chars = 0
+  if (msg.content) chars += msg.content.length
+  if (msg.toolCalls) chars += JSON.stringify(msg.toolCalls).length
+  if (msg.toolResults) chars += JSON.stringify(msg.toolResults).length
+  return Math.ceil(chars / 4)
+}
+
+export function parseWorkingMemory(
+  text: string
+): Record<string, unknown> | null {
+  const match = text.match(/<working_memory>([\s\S]*?)<\/working_memory>/)
+  if (!match) return null
+  try {
+    return JSON.parse(match[1].trim())
+  } catch {
+    return null
+  }
+}
+
+export function stripWorkingMemoryTags(text: string): string {
+  return text.replace(/<working_memory>[\s\S]*?<\/working_memory>/g, '').trim()
+}