@ossy/media-tasks 3.15.0 → 3.17.0

package/src/visual-content-descriptors.spec.js

@@ -0,0 +1,130 @@
+import {
+  DEFAULT_VCD_MODEL,
+  getVisualContentDescriptors,
+  toImageDataUrl,
+} from './visual-content-descriptors.js'
+
+describe('toImageDataUrl', () => {
+  it('encodes buffer as a data URL with mime type', () => {
+    const buf = Buffer.from([0x89, 0x50, 0x4e, 0x47])
+    expect(toImageDataUrl(buf, 'image/png')).toBe(
+      `data:image/png;base64,${buf.toString('base64')}`,
+    )
+  })
+
+  it('rejects empty buffers', () => {
+    expect(() => toImageDataUrl(Buffer.alloc(0))).toThrow(/image buffer/)
+  })
+})
+
+describe('getVisualContentDescriptors', () => {
+  it('calls chat.completions once with json_object and OpenRouter model slug', async () => {
+    const calls = []
+    const client = {
+      chat: {
+        completions: {
+          create: async (args) => {
+            calls.push(args)
+            return {
+              choices: [{
+                message: {
+                  content: JSON.stringify({
+                    title: 'Sunset',
+                    description: 'Orange sky over water',
+                    tags: ['sunset', 'nature'],
+                    alt: 'Sunset over water',
+                  }),
+                },
+              }],
+            }
+          },
+        },
+      },
+    }
+
+    const dataUrl = 'data:image/jpeg;base64,abc'
+    await expect(getVisualContentDescriptors(client, dataUrl))
+      .resolves.toEqual({
+        title: 'Sunset',
+        description: 'Orange sky over water',
+        tags: ['sunset', 'nature'],
+        alt: 'Sunset over water',
+        usage: { model: DEFAULT_VCD_MODEL, promptTokens: null, completionTokens: null, totalTokens: null },
+      })
+
+    expect(calls).toHaveLength(1)
+    expect(calls[0]).toMatchObject({
+      model: DEFAULT_VCD_MODEL,
+      response_format: { type: 'json_object' },
+      messages: [
+        { role: 'system', content: expect.any(String) },
+        {
+          role: 'user',
+          content: [{ type: 'image_url', image_url: { url: dataUrl } }],
+        },
+      ],
+    })
+  })
+
+  it('normalizes missing fields and non-string tags', async () => {
+    const client = {
+      chat: {
+        completions: {
+          create: async () => ({
+            choices: [{ message: { content: '{"title":1,"tags":["ok",2]}' } }],
+          }),
+        },
+      },
+    }
+    await expect(getVisualContentDescriptors(client, 'data:image/png;base64,x'))
+      .resolves.toEqual({
+        title: null,
+        description: null,
+        tags: ['ok'],
+        alt: null,
+        usage: { model: DEFAULT_VCD_MODEL, promptTokens: null, completionTokens: null, totalTokens: null },
+      })
+  })
+
+  it('captures token usage and the model that actually answered', async () => {
+    const client = {
+      chat: {
+        completions: {
+          create: async () => ({
+            model: 'openai/gpt-4o-mini-2024-07-18',
+            usage: { prompt_tokens: 120, completion_tokens: 40, total_tokens: 160 },
+            choices: [{ message: { content: '{"title":"Sunset"}' } }],
+          }),
+        },
+      },
+    }
+    await expect(getVisualContentDescriptors(client, 'data:image/png;base64,x'))
+      .resolves.toMatchObject({
+        usage: {
+          model: 'openai/gpt-4o-mini-2024-07-18',
+          promptTokens: 120,
+          completionTokens: 40,
+          totalTokens: 160,
+        },
+      })
+  })
+
+  it('rejects missing client or imageUrl', async () => {
+    await expect(getVisualContentDescriptors(null, 'data:image/png;base64,x'))
+      .rejects.toThrow(/openrouter client/)
+    await expect(getVisualContentDescriptors({ chat: { completions: { create: async () => ({}) } } }, ''))
+      .rejects.toThrow(/imageUrl/)
+  })
+
+  it('rejects non-JSON model content', async () => {
+    const client = {
+      chat: {
+        completions: {
+          create: async () => ({ choices: [{ message: { content: 'not-json' } }] }),
+        },
+      },
+    }
+    await expect(getVisualContentDescriptors(client, 'data:image/png;base64,x'))
+      .rejects.toThrow(/non-JSON/)
+  })
+})

package/src/visual-content-descriptors.task.js

@@ -1,8 +1,30 @@
-import { GetResource, UpdateResourceName, UpdateResourceContent } from '@ossy/resources'
-import { OpenAi } from './openai.integration.js'
+import { createLogger } from '@ossy/observability'
+import { StorageClient } from '@ossy/platform'
+import { taskArtifactObjectKey } from '@ossy/platform/storage-keys'
+import { assertLlmUsageBudget, resolveWorkspaceIdForBudget } from '@ossy/platform/metering'
+import { GetResource, placeTaskArtifacts } from '@ossy/resources'
+import { loadSourceBuffer } from './load-source-buffer.js'
+import {
+  DEFAULT_VCD_MODEL,
+  getVisualContentDescriptors,
+  toImageDataUrl,
+} from './visual-content-descriptors.js'
+import { resolveConfiguredPlacements } from './resolve-configured-placement.js'
+
+const log = createLogger('media-tasks')
+
+const TASK_ID = '@ossy/media-tasks/tasks/visual-content-descriptors'
+const SCHEMA_ID = '@ossy/media-tasks/schema/visual-content-descriptors'
+const CONCEPT = 'visual-content-descriptors'
+const OUTPUT = {
+  name: 'descriptors',
+  kind: 'location',
+  schemaId: SCHEMA_ID,
+  contentType: 'application/json',
+}
 
 export const metadata = {
-  id: '@ossy/media-tasks/tasks/visual-content-descriptors',
+  id: TASK_ID,
   triggers: [
     {
       type: '@ossy/platform/schema/file',
@@ -15,20 +37,84 @@ export const metadata = {
       match: { 'content.ContentType': 'video/*' },
     },
   ],
+  outputs: [OUTPUT],
+  inputs: [
+    {
+      name: 'model',
+      type: 'text',
+      default: DEFAULT_VCD_MODEL,
+      sources: ['payload', 'config'],
+      description: 'OpenRouter model slug',
+    },
+  ],
+  configurable: true,
 }
 
-export async function run ({ event, sdk }) {
+export async function run ({ event, payload, req, sdk, integrations, inputs }) {
   const resourceId = event.resourceId
 
+  log.info(`[media-tasks/tasks/visual-content-descriptors] starting for resource ${resourceId}`)
+
+  // Fail closed before spending anything — ADR 0017's first QuotaPolicy.
+  await assertLlmUsageBudget({
+    workspaceId: resolveWorkspaceIdForBudget({ event, payload, req }),
+  })
+
+  const openrouter = integrations?.get?.('openrouter')
+  if (!openrouter) {
+    log.warn('[media-tasks/tasks/visual-content-descriptors] openrouter integration not connected')
+    throw Object.assign(new Error('OpenRouter integration not available'), { status: 503 })
+  }
+
   const resource = await sdk.invoke(GetResource, { resourceId })
-  const visualContentDescriptors = await OpenAi.getVisualContentDescriptors(resource.content.src)
-
-  await sdk.invoke(UpdateResourceName, { id: resource.id, name: visualContentDescriptors.title })
-  await sdk.invoke(UpdateResourceContent, {
-    id: resource.id,
-    content: {
-      ...resource.content,
-      ...visualContentDescriptors,
-    },
+  const key = resource?.content?.Key
+  if (!key) {
+    throw new Error(`[media-tasks/tasks/visual-content-descriptors] missing storage key for ${resourceId}`)
+  }
+
+  const contentType = resource?.content?.ContentType || 'application/octet-stream'
+  if (!String(contentType).startsWith('image/')) {
+    throw new Error(
+      `[media-tasks/tasks/visual-content-descriptors] unsupported ContentType ${contentType} for ${resourceId}`,
+    )
+  }
+
+  const sourceBuffer = await loadSourceBuffer(key)
+  if (!sourceBuffer) {
+    throw new Error(`[media-tasks/tasks/visual-content-descriptors] missing file bytes for ${resourceId}`)
+  }
+
+  // Inline bytes — OpenRouter cannot fetch localhost / auth-gated storage URLs.
+  const imageUrl = toImageDataUrl(sourceBuffer, contentType)
+  // `usage` is metering-only — never persisted on the artifact/placed document.
+  const { usage, ...body } = await getVisualContentDescriptors(openrouter, imageUrl, {
+    model: inputs?.model,
+  })
+  const objectKey = taskArtifactObjectKey(resourceId, CONCEPT, OUTPUT.name)
+  await StorageClient.save(objectKey, Buffer.from(JSON.stringify(body), 'utf8'))
+  log.info(`[media-tasks/tasks/visual-content-descriptors] saved ${objectKey}`)
+
+  const placementLocations = await resolveConfiguredPlacements(sdk, TASK_ID, OUTPUT.name)
+  const placed = await placeTaskArtifacts({
+    sdk,
+    sourceResource: resource,
+    taskId: TASK_ID,
+    output: OUTPUT,
+    body,
+    artifactKey: objectKey,
+    placementLocations,
   })
+  if (placed.length) {
+    log.info(`[media-tasks/tasks/visual-content-descriptors] placed ${placed.length} resource(s)`)
+  } else {
+    log.info(`[media-tasks/tasks/visual-content-descriptors] artifact only (no placement location configured)`)
+  }
+  return {
+    output: OUTPUT.name,
+    key: objectKey,
+    descriptors: body,
+    placedResourceIds: placed.map((r) => r.id),
+    placedResourceId: placed[0]?.id ?? null,
+    usage,
+  }
 }

package/src/openai.integration.js

@@ -1,74 +0,0 @@
-import OpenAiSDK from 'openai'
-
-export const id = 'openai'
-
-export const credentials = ['OPENAI_API_KEY']
-
-/**
- * Returns an OpenAI client configured from env vars.
- * Tasks receive this via `integrations.get('openai')`.
- *
- * @param {{ env: NodeJS.ProcessEnv }} opts
- * @returns {import('openai').OpenAI}
- */
-export async function connect({ env }) {
-  return new OpenAiSDK({
-    apiKey: env.OPENAI_API_KEY,
-    organization: env.OPENAI_ORGANIZATION,
-  })
-}
-
-let _openai = null
-function getOpenAi() {
-  if (!_openai) _openai = new OpenAiSDK({
-    apiKey: process.env.OPENAI_API_KEY,
-    organization: process.env.OPENAI_ORGANIZATION,
-  })
-  return _openai
-}
-
-export class OpenAi {
-  static async getVisualContentDescriptors(imageSrc) {
-    const systemPrompt = `
-            You will be provided with images, and your task is to create a json object including the following properties:
-                -title: Short attention grabbing title that describes the image
-                -description: A short text describing the image, suitable for SEO purposes
-                -tags: an array of tags describing the image that will be used for categorization and SEO purposes
-                -alt: A short text that describes the image, suitable for alt text
-        `
-
-    const response = await getOpenAi().chat.completions.create({
-      model: 'gpt-4o-mini',
-      max_tokens: 4000,
-      messages: [
-        { role: 'system', content: systemPrompt },
-        { role: 'user', content: [{ type: 'image_url', image_url: { url: imageSrc } }] },
-      ],
-    })
-
-    const parsedResponse = await OpenAi.ensureJSONResponse(response)
-    const visualDescriptors = JSON.parse(
-      parsedResponse?.choices?.[0]?.message?.content ?? '{}'
-    )
-    return visualDescriptors
-  }
-
-  static ensureJSONResponse(response) {
-    const systemPrompt = `
-            You will be recieving a text string that contains a json object.
-            Your task is to parse the text string and convert the response into a json object.
-        `
-
-    const textString = response?.choices?.[0]?.message?.content
-
-    return getOpenAi().chat.completions.create({
-      model: 'gpt-4-turbo-preview',
-      max_tokens: 4000,
-      response_format: { type: 'json_object' },
-      messages: [
-        { role: 'system', content: systemPrompt },
-        { role: 'user', content: textString ?? '' },
-      ],
-    })
-  }
-}