@nekro-nxt/contracts 0.5.0 → 0.7.0

package/src/extension-capabilities.ts

@@ -6,8 +6,19 @@ import { z } from 'zod'
  * ships; never reuse a level for a different meaning. Level 7 is Manifest V7 (host instance + agent attachments); level 8 adds the asset library, image info, tool result
  * images, the search index and host-layer model calls. Level 9 adds the structured logger and explicit chat commands. Level 10 adds version lineage, the data version with its
  * migration hook and administrator-only commands. Level 11 lists the pictures of an inbound message to inbound hooks.
+ * Level 12 adds credential placeholders (`secrets.placeholder`) that `http.fetch` fills for the credential's domains.
+ * Level 13 adds outbound processing (`harness.onOutbound`, `permissions.agent.outbound`).
+ * Level 14 adds work files (`files`), reading Asset bytes (`assets.read`), high-detail tool images (`toolImages`),
+ * long-running tools (`timeoutSeconds`, `execute(args, { signal, progress })`), media processing (`media`), command
+ * profiles (`commands`) and speech to text (`speech`).
  */
-export const EXTENSION_SDK_LEVEL = 11
+export const EXTENSION_SDK_LEVEL = 14
+
+/** Work files, Asset reads, high-detail tool images, tool time limits, media, commands and speech need this level. */
+export const EXTENSION_FILES_SDK_LEVEL = 14
+
+/** Outbound processing needs this level; packages that declare `permissions.agent.outbound` declare it. */
+export const EXTENSION_OUTBOUND_SDK_LEVEL = 13
 
 /** Lineage, `dataVersion` and administrator-only commands need this level; packages that carry them declare it. */
 export const EXTENSION_GOVERNANCE_SDK_LEVEL = 10
@@ -23,6 +34,9 @@ export const EXTENSION_SDK_RELEASES: readonly { readonly level: number; readonly
   { level: 9, version: '0.6.0' },
   { level: 10, version: '0.6.0' },
   { level: 11, version: '0.6.0' },
+  { level: 12, version: '0.7.0' },
+  { level: 13, version: '0.7.0' },
+  { level: 14, version: '0.8.0' },
 ]
 
 /** The oldest NekroNXT version that supports `level`; undefined for a level no release supports yet. */
@@ -128,6 +142,32 @@ export const ExtensionInboundHookCapabilitySchema = z
   .strict()
 export type ExtensionInboundHookCapability = z.output<typeof ExtensionInboundHookCapabilitySchema>
 
+export const EXTENSION_OUTBOUND_MAX_TIMEOUT_MS = 15_000
+
+/**
+ * Outbound processing of the agent's own messages. `read` sees each message after it was delivered; `transform` may
+ * replace its content or append messages before it is sent; `cancel` may stop it. `onTimeout` decides what happens
+ * when the handler times out or throws before sending: `send` (default) sends the message as it is.
+ */
+export const ExtensionOutboundCapabilitySchema = z
+  .object({
+    read: z.literal(true).optional(),
+    transform: z.literal(true).optional(),
+    cancel: z.literal(true).optional(),
+    timeoutMs: z.number().int().min(100).max(EXTENSION_OUTBOUND_MAX_TIMEOUT_MS),
+    onTimeout: z.enum(['send', 'cancel']).optional(),
+  })
+  .strict()
+  .refine(
+    (value) => value.read === true || value.transform === true || value.cancel === true,
+    'outbound 至少声明 read、transform 或 cancel 中的一项。',
+  )
+export type ExtensionOutboundCapability = z.output<typeof ExtensionOutboundCapabilitySchema>
+
+/** Whether the extension takes part before sending (and so in the agent's processing order). */
+export const outboundProcessesBeforeSend = (outbound: ExtensionOutboundCapability | undefined): boolean =>
+  outbound?.transform === true || outbound?.cancel === true
+
 export const EXTENSION_JOB_MAX_RUNTIME = 50
 
 export const ExtensionDeclaredJobSchema = z
@@ -192,17 +232,92 @@ export type ExtensionAssetLibraryCapability = z.output<typeof ExtensionAssetLibr
 
 /**
  * `write` creates Assets in the current channel; `library` keeps Assets in the extension's own library so they can be
- * used in any channel later.
+ * used in any channel later; `read` reads the bytes of Assets the current channel can see or the library holds.
  */
 export const ExtensionAssetsCapabilitySchema = z
   .object({
     write: z.literal(true).optional(),
     library: ExtensionAssetLibraryCapabilitySchema.optional(),
+    read: z.literal(true).optional(),
   })
   .strict()
-  .refine((value) => value.write !== undefined || value.library !== undefined, 'assets 至少声明 write 或 library。')
+  .refine(
+    (value) => value.write !== undefined || value.library !== undefined || value.read !== undefined,
+    'assets 至少声明 write、library 或 read。',
+  )
 export type ExtensionAssetsCapability = z.output<typeof ExtensionAssetsCapabilitySchema>
 
+export const EXTENSION_FILES_DEFAULT_TEMP_QUOTA_BYTES = 1024 * 1024 * 1024
+export const EXTENSION_FILES_DEFAULT_CACHE_QUOTA_BYTES = 1024 * 1024 * 1024
+export const EXTENSION_FILES_MAX_QUOTA_BYTES = 16 * 1024 * 1024 * 1024
+
+const FilesQuotaSchema = z
+  .number()
+  .int()
+  .min(1024 * 1024)
+  .max(EXTENSION_FILES_MAX_QUOTA_BYTES)
+
+/**
+ * Work files passed between services by `fileId`. Temporary files expire a day after they are written; cache entries
+ * stay until the cache quota is full, then the least recently used go first.
+ */
+export const ExtensionFilesCapabilitySchema = z
+  .object({ tempQuotaBytes: FilesQuotaSchema.optional(), cacheQuotaBytes: FilesQuotaSchema.optional() })
+  .strict()
+export type ExtensionFilesCapability = z.output<typeof ExtensionFilesCapabilitySchema>
+
+/** Tool result pictures with `detail: 'high'` reach the model at up to 2048 px on the long side. */
+export const ExtensionToolImagesCapabilitySchema = z.object({ highDetail: z.literal(true) }).strict()
+export type ExtensionToolImagesCapability = z.output<typeof ExtensionToolImagesCapabilitySchema>
+
+/**
+ * Add-ons (附加功能) whose capability an extension can use. The ids match `AddonIdSchema` in host-api; they are
+ * repeated here so manifests validate without it.
+ */
+export type ExtensionAddonId = 'media' | 'speech'
+
+/**
+ * Command profiles extensions may run (`nxt.commands`, 附加功能 F7), and the add-on that provides each program. A new
+ * program needs a profile on the server and a row here.
+ */
+export const ExtensionCommandProfileIdSchema = z.enum(['ffmpeg', 'ffprobe'], {
+  message: '命令档案只能是 ffmpeg、ffprobe。',
+})
+export type ExtensionCommandProfileId = z.output<typeof ExtensionCommandProfileIdSchema>
+
+export const EXTENSION_COMMAND_PROFILES: Readonly<Record<ExtensionCommandProfileId, ExtensionAddonId>> = {
+  ffmpeg: 'media',
+  ffprobe: 'media',
+}
+
+/** Programs run through their command profiles: confined to the call's own files, without network. */
+export const ExtensionCommandsCapabilitySchema = z
+  .object({
+    profiles: z
+      .array(ExtensionCommandProfileIdSchema)
+      .min(1)
+      .max(ExtensionCommandProfileIdSchema.options.length)
+      .refine((value) => new Set(value).size === value.length, '命令档案不能重复。'),
+  })
+  .strict()
+export type ExtensionCommandsCapability = z.output<typeof ExtensionCommandsCapabilitySchema>
+
+export const EXTENSION_SPEECH_MAX_CALLS_PER_MINUTE = 60
+export const EXTENSION_SPEECH_DEFAULT_CALLS_PER_MINUTE = 10
+
+/** Speech to text with the provider chosen in 设置 → 附加功能, which may be a cloud service. */
+export const ExtensionSpeechCapabilitySchema = z
+  .object({
+    maxCallsPerMinute: z
+      .number()
+      .int()
+      .min(1)
+      .max(EXTENSION_SPEECH_MAX_CALLS_PER_MINUTE)
+      .default(EXTENSION_SPEECH_DEFAULT_CALLS_PER_MINUTE),
+  })
+  .strict()
+export type ExtensionSpeechCapability = z.output<typeof ExtensionSpeechCapabilitySchema>
+
 const IndexNameSchema = z
   .string()
   .trim()
@@ -255,10 +370,14 @@ export const ExtensionModelsCapabilitySchema = z
   .strict()
 export type ExtensionModelsCapability = z.output<typeof ExtensionModelsCapabilitySchema>
 
-/** A header or environment value: literal text, or the current value of a `meta.role: 'secret'` config field. */
+/**
+ * A header or environment value: literal text, the current value of a `meta.role: 'secret'` config field, or a named
+ * credential (by id or name) that the agent may use.
+ */
 export const ExtensionMcpValueSchema = z.union([
   z.string().max(4096),
   z.object({ secret: z.string().trim().min(1).max(64) }).strict(),
+  z.object({ credential: z.string().trim().min(1).max(64) }).strict(),
 ])
 export type ExtensionMcpValue = z.output<typeof ExtensionMcpValueSchema>
 
@@ -315,6 +434,8 @@ const McpFormValueSchema = z
     name: z.string().regex(/^[A-Za-z_][A-Za-z0-9_-]{0,127}$/u, '名称只能包含字母、数字、下划线和连字符。'),
     value: z.string().max(4096),
     secret: z.boolean().default(false),
+    /** Takes the value from this named credential instead; `value` and `secret` are then ignored. */
+    credential: z.string().trim().min(1).max(64).optional(),
   })
   .strict()
 
@@ -344,7 +465,7 @@ export const mcpSecretFields = (mcp: ExtensionMcpCapability | undefined): readon
   ...new Set(
     (mcp?.servers ?? []).flatMap((server) =>
       Object.values(server.transport === 'stdio' ? server.env : server.headers).flatMap((value) =>
-        typeof value === 'string' ? [] : [value.secret],
+        typeof value === 'string' || !('secret' in value) ? [] : [value.secret],
       ),
     ),
   ),
@@ -365,6 +486,12 @@ export const ExtensionCapabilitiesSchema = z
     network: ExtensionNetworkCapabilitySchema.optional(),
     storage: ExtensionStorageCapabilitySchema.optional(),
     assets: ExtensionAssetsCapabilitySchema.optional(),
+    files: ExtensionFilesCapabilitySchema.optional(),
+    toolImages: ExtensionToolImagesCapabilitySchema.optional(),
+    /** `nxt.media`: probe, frames, audio and convert on the media add-on. */
+    media: z.literal(true).optional(),
+    commands: ExtensionCommandsCapabilitySchema.optional(),
+    speech: ExtensionSpeechCapabilitySchema.optional(),
     index: ExtensionIndexCapabilitySchema.optional(),
     /** Host layer only; the agent layer uses `llm`, which runs on the agent's own model. */
     models: ExtensionModelsCapabilitySchema.optional(),
@@ -373,6 +500,7 @@ export const ExtensionCapabilitiesSchema = z
       .strict()
       .optional(),
     inboundHook: ExtensionInboundHookCapabilitySchema.optional(),
+    outbound: ExtensionOutboundCapabilitySchema.optional(),
     jobs: ExtensionJobsCapabilitySchema.optional(),
     platform: ExtensionPlatformCapabilitySchema.optional(),
     llm: ExtensionLlmCapabilitySchema.optional(),
@@ -397,8 +525,19 @@ export const HostLayerCapabilitiesSchema = z
       .object({ quotaBytes: z.number().int().min(1024).max(EXTENSION_STORAGE_MAX_QUOTA_BYTES).optional() })
       .strict()
       .optional(),
-    /** The same library agent attachments use; the host instance lists and imports into it. */
-    assets: z.object({ library: ExtensionAssetLibraryCapabilitySchema }).strict().optional(),
+    /**
+     * The same library agent attachments use; the host instance lists and imports into it. `read` reads the bytes of
+     * library Assets.
+     */
+    assets: z
+      .object({ library: ExtensionAssetLibraryCapabilitySchema.optional(), read: z.literal(true).optional() })
+      .strict()
+      .refine((value) => value.library !== undefined || value.read !== undefined, 'assets 至少声明 library 或 read。')
+      .optional(),
+    files: ExtensionFilesCapabilitySchema.optional(),
+    media: z.literal(true).optional(),
+    commands: ExtensionCommandsCapabilitySchema.optional(),
+    speech: ExtensionSpeechCapabilitySchema.optional(),
     /** The same collections agent attachments declare; pages search and maintain them. */
     index: ExtensionIndexCapabilitySchema.optional(),
     models: ExtensionModelsCapabilitySchema.optional(),
@@ -420,7 +559,18 @@ export const hostLayerAsCapabilities = (host: HostLayerCapabilities | undefined)
                 ...(host.storage.quotaBytes === undefined ? {} : { quotaBytes: host.storage.quotaBytes }),
               },
             }),
-        ...(host.assets === undefined ? {} : { assets: { library: host.assets.library } }),
+        ...(host.assets === undefined
+          ? {}
+          : {
+              assets: {
+                ...(host.assets.library === undefined ? {} : { library: host.assets.library }),
+                ...(host.assets.read === undefined ? {} : { read: host.assets.read }),
+              },
+            }),
+        ...(host.files === undefined ? {} : { files: host.files }),
+        ...(host.media === undefined ? {} : { media: host.media }),
+        ...(host.commands === undefined ? {} : { commands: host.commands }),
+        ...(host.speech === undefined ? {} : { speech: host.speech }),
         ...(host.index === undefined ? {} : { index: host.index }),
         ...(host.models === undefined ? {} : { models: host.models }),
       }
@@ -459,6 +609,19 @@ export const extensionCapabilitiesExpand = (
   if (next.storage?.scopes.some((scope) => !(previous?.storage?.scopes ?? []).includes(scope))) return true
   if (next.assets?.write !== undefined && previous?.assets?.write === undefined) return true
   if (next.assets?.library !== undefined && previous?.assets?.library === undefined) return true
+  if (next.assets?.read !== undefined && previous?.assets?.read === undefined) return true
+  if (next.files !== undefined && previous?.files === undefined) return true
+  if (next.toolImages !== undefined && previous?.toolImages === undefined) return true
+  if (next.media !== undefined && previous?.media === undefined) return true
+  if ((next.commands?.profiles ?? []).some((profile) => !(previous?.commands?.profiles ?? []).includes(profile))) {
+    return true
+  }
+  if (
+    next.speech !== undefined &&
+    (previous?.speech === undefined || next.speech.maxCallsPerMinute > previous.speech.maxCallsPerMinute)
+  ) {
+    return true
+  }
   if (
     next.models !== undefined &&
     (previous?.models === undefined ||
@@ -484,6 +647,18 @@ export const extensionCapabilitiesExpand = (
   ) {
     return true
   }
+  const outbound = next.outbound
+  const previousOutbound = previous?.outbound
+  if (
+    outbound !== undefined &&
+    (previousOutbound === undefined ||
+      (outbound.read === true && previousOutbound.read !== true) ||
+      (outbound.transform === true && previousOutbound.transform !== true) ||
+      (outbound.cancel === true && previousOutbound.cancel !== true) ||
+      (outbound.onTimeout === 'cancel' && previousOutbound.onTimeout !== 'cancel'))
+  ) {
+    return true
+  }
   if (
     next.jobs?.runtime !== undefined &&
     (previous?.jobs?.runtime === undefined || next.jobs.runtime.maxActive > previous.jobs.runtime.maxActive)
@@ -516,6 +691,25 @@ export interface ExtensionCapabilitySummaryItem {
   readonly risk: ExtensionCapabilityRisk
   readonly label: string
   readonly detail?: string
+  /** For a high-risk capability with limits: what it stays within, shown where the user accepts the risk. */
+  readonly scope?: string
+}
+
+/**
+ * Add-ons the declared capabilities need (附加功能 F13), in a stable order: `media` needs 媒体处理, `speech` needs
+ * 语音识别, and each command profile needs the add-on that provides its program. Pass both layers, host-layer ones
+ * through {@link hostLayerAsCapabilities}.
+ */
+export const requiredExtensionAddons = (
+  ...layers: readonly (ExtensionCapabilities | undefined)[]
+): readonly ExtensionAddonId[] => {
+  const needed = new Set<ExtensionAddonId>()
+  for (const capabilities of layers) {
+    if (capabilities?.media !== undefined) needed.add('media')
+    if (capabilities?.speech !== undefined) needed.add('speech')
+    for (const profile of capabilities?.commands?.profiles ?? []) needed.add(EXTENSION_COMMAND_PROFILES[profile])
+  }
+  return (['media', 'speech'] as const).filter((id) => needed.has(id))
 }
 
 /**
@@ -565,6 +759,49 @@ export const summarizeExtensionCapabilities = (
       detail: `最多 ${formatLibraryQuota(capabilities.assets.library.quotaBytes ?? EXTENSION_ASSET_LIBRARY_DEFAULT_QUOTA_BYTES)}`,
     })
   }
+  if (capabilities.assets?.read !== undefined) {
+    items.push(
+      options.layer === 'host'
+        ? { key: 'assets.read', risk: 'normal', label: '读取扩展资源库里的文件' }
+        : { key: 'assets.read', risk: 'sensitive', label: '读取当前频道里的图片和文件' },
+    )
+  }
+  if (capabilities.files !== undefined) {
+    items.push({
+      key: 'files',
+      risk: 'normal',
+      label: '在本机存放临时文件和缓存',
+      detail: `临时文件最多 ${formatLibraryQuota(capabilities.files.tempQuotaBytes ?? EXTENSION_FILES_DEFAULT_TEMP_QUOTA_BYTES)}，缓存最多 ${formatLibraryQuota(capabilities.files.cacheQuotaBytes ?? EXTENSION_FILES_DEFAULT_CACHE_QUOTA_BYTES)}`,
+    })
+  }
+  if (capabilities.toolImages !== undefined) {
+    items.push({
+      key: 'toolImages',
+      risk: 'normal',
+      label: '以高细节向智能体展示图片',
+      detail: '计入智能体用量',
+    })
+  }
+  if (capabilities.media !== undefined) {
+    items.push({ key: 'media', risk: 'normal', label: '处理音频、视频和图片' })
+  }
+  if (capabilities.commands !== undefined) {
+    items.push({
+      key: 'commands',
+      risk: 'high',
+      label: '在本机运行程序',
+      detail: capabilities.commands.profiles.join('、'),
+      scope: '只能读写交给它的文件，不能联网',
+    })
+  }
+  if (capabilities.speech !== undefined) {
+    items.push({
+      key: 'speech',
+      risk: 'sensitive',
+      label: '把音频转成文字',
+      detail: `音频可能发给「设置 → 附加功能」里选择的语音识别服务，每分钟最多 ${capabilities.speech.maxCallsPerMinute} 次`,
+    })
+  }
   if (capabilities.index !== undefined) {
     items.push({
       key: 'index',
@@ -598,6 +835,21 @@ export const summarizeExtensionCapabilities = (
       ...(effects === '' ? {} : { detail: `可以${effects}` }),
     })
   }
+  const outbound = capabilities.outbound
+  if (outbound !== undefined) {
+    if (outbound.transform === true) {
+      items.push({ key: 'outbound.transform', risk: 'high', label: '在智能体的消息发出前修改内容或追加消息' })
+    }
+    if (outbound.cancel === true) {
+      items.push({ key: 'outbound.cancel', risk: 'high', label: '取消智能体要发出的消息' })
+    }
+    if (outbound.read === true) {
+      items.push({ key: 'outbound.read', risk: 'sensitive', label: '查看智能体发出的每条消息' })
+    }
+    if (outbound.onTimeout === 'cancel' && (outbound.transform === true || outbound.cancel === true)) {
+      items.push({ key: 'outbound.onTimeout', risk: 'high', label: '这个扩展出错时，智能体的消息会发不出去' })
+    }
+  }
   const jobs = capabilities.jobs
   if (jobs?.declared !== undefined && jobs.declared.length > 0) {
     items.push({

package/src/extension-ui.ts

@@ -4,6 +4,8 @@ import {
   HostLayerCapabilitiesSchema,
   hostLayerAsCapabilities,
   EXTENSION_SDK_BASE_LEVEL,
+  EXTENSION_FILES_SDK_LEVEL,
+  EXTENSION_OUTBOUND_SDK_LEVEL,
 } from './extension-capabilities.js'
 import { ExtensionIdSchema, ExtensionRevisionIdSchema, HostUiPageInstanceIdSchema } from './domain.js'
 import { ConfigSchemaDocumentSchema } from './config-schema.js'
@@ -548,12 +550,25 @@ export type ExtensionDataHookName = keyof typeof EXTENSION_DATA_HOOK_PERMISSIONS
 export const requiredExtensionSdkLevel = (
   permissions: ExtensionPermissions | undefined,
 ): { readonly level: number; readonly capabilities: readonly string[] } => {
+  const level14 = [
+    permissions?.host?.files === undefined && permissions?.agent?.files === undefined ? undefined : '工作文件',
+    permissions?.host?.assets?.read === undefined && permissions?.agent?.assets?.read === undefined
+      ? undefined
+      : '读取频道文件',
+    permissions?.agent?.toolImages === undefined ? undefined : '高细节图片',
+    permissions?.host?.media === undefined && permissions?.agent?.media === undefined ? undefined : '媒体处理',
+    permissions?.host?.commands === undefined && permissions?.agent?.commands === undefined ? undefined : '运行程序',
+    permissions?.host?.speech === undefined && permissions?.agent?.speech === undefined ? undefined : '语音识别',
+  ].filter((name): name is string => name !== undefined)
+  if (level14.length > 0) return { level: EXTENSION_FILES_SDK_LEVEL, capabilities: level14 }
   const level8 = [
-    permissions?.host?.assets === undefined ? undefined : '扩展资源库',
+    permissions?.host?.assets?.library === undefined ? undefined : '扩展资源库',
     permissions?.agent?.assets?.library === undefined ? undefined : '扩展资源库',
     permissions?.host?.index === undefined && permissions?.agent?.index === undefined ? undefined : '检索索引',
     permissions?.host?.models === undefined ? undefined : '本机层模型调用',
   ].filter((name): name is string => name !== undefined)
+  if (permissions?.agent?.outbound !== undefined)
+    return { level: EXTENSION_OUTBOUND_SDK_LEVEL, capabilities: ['出站处理'] }
   return level8.length > 0
     ? { level: 8, capabilities: [...new Set(level8)] }
     : { level: EXTENSION_SDK_BASE_LEVEL, capabilities: [] }