@llumiverse/drivers 0.15.0 → 0.16.0

package/src/adobe/firefly.ts

@@ -0,0 +1,207 @@
+import {
+    AbstractDriver, AIModel,
+    Completion,
+    CompletionChunk, DriverOptions, EmbeddingsOptions,
+    EmbeddingsResult,
+    ExecutionOptions,
+    ImageGeneration,
+    ModelSearchPayload,
+    PromptSegment
+} from "@llumiverse/core";
+
+interface FireflyImageSource {
+    url?: string;
+    uploadId?: string;
+}
+
+interface FireflyImageReference {
+    source: FireflyImageSource;
+}
+
+interface FireflyStyle {
+    presets?: string[];
+    strength?: number;
+    imageReference?: FireflyImageReference;
+}
+
+interface FireflyStructure {
+    strength: number;
+    imageReference: FireflyImageReference;
+}
+
+interface FireflySize {
+    width: number;
+    height: number;
+}
+
+interface FireflyGenerateRequest {
+    numVariations?: number;
+    seeds?: number[];
+    size?: FireflySize;
+    prompt: string;
+    negativePrompt?: string;
+    contentClass?: 'photo' | 'art' | 'graphic';
+    visualIntensity?: number;
+    style?: FireflyStyle;
+    promptBiasingLocaleCode?: string;
+    tileable?: boolean;
+    structure?: FireflyStructure;
+}
+
+interface FireflyOutput {
+    seed: number;
+    image: {
+        url: string;
+    };
+}
+
+interface FireflyGenerateResponse {
+    size: FireflySize;
+    outputs: FireflyOutput[];
+    promptHasDeniedWords?: boolean;
+    promptHasBlockedArtists?: boolean;
+    contentClass?: string;
+}
+
+export interface FireflyDriverOptions extends DriverOptions {
+    /**
+     * Adobe Firefly API key
+     */
+    apiKey: string;
+
+    /**
+     * Optional API endpoint override
+     */
+    endpoint?: string;
+}
+
+export class FireflyDriver extends AbstractDriver<FireflyDriverOptions> {
+    static PROVIDER = "firefly";
+    provider = FireflyDriver.PROVIDER;
+
+    private readonly endpoint: string;
+
+    constructor(options: FireflyDriverOptions) {
+        super(options);
+        
+        if (!options.apiKey) {
+            throw new Error("No API key provided for Firefly driver");
+        }
+
+        this.endpoint = options.endpoint || "https://firefly-api.adobe.io/v3";
+    }
+
+    async requestTextCompletion(_prompt: string, _options: ExecutionOptions): Promise<Completion> {
+        throw new Error("Text completion not supported by Firefly");
+    }
+
+    async requestTextCompletionStream(_prompt: string, _options: ExecutionOptions): Promise<AsyncIterable<CompletionChunk>> {
+        throw new Error("Text completion streaming not supported by Firefly");
+    }
+
+    async requestImageGeneration(segments: PromptSegment[], options: ExecutionOptions): Promise<Completion<ImageGeneration>> {
+        this.logger.debug(`[${this.provider}] Generating image with model ${options.model}`);
+        const prompt = segments.map(s => s.content).join("\n\n");
+    
+
+        try {
+            const payload: FireflyGenerateRequest = {
+                prompt: prompt as string,
+            };
+
+            const response = await fetch(`${this.endpoint}/images/generate`, {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json',
+                    'x-api-key': this.options.apiKey,
+                },
+                body: JSON.stringify(payload)
+            });
+
+            if (!response.ok) {
+                const error = await response.json();
+                throw new Error(`Firefly API error: ${error.message || response.statusText}`);
+            }
+
+            const result: FireflyGenerateResponse = await response.json();
+
+            if (result.promptHasDeniedWords || result.promptHasBlockedArtists) {
+                return {
+                    result: {},
+                    error: {
+                        message: "Prompt contains denied words or blocked artists",
+                        code: "content_policy_violation"
+                    }
+                };
+            }
+
+            return {
+                result: {
+                    images: result.outputs.map(output => output.image.url)
+                }
+            };
+
+        } catch (error: any) {
+            this.logger.error("[Firefly] Image generation failed", { error });
+            return {
+                result: {},
+                error: {
+                    message: error.message,
+                    code: error.code || 'GENERATION_FAILED'
+                }
+            };
+        }
+    }
+
+    mapSize(size?: string): FireflySize {
+        // Default to 1024x1024 if no size specified
+        if (!size) return { width: 1024, height: 1024 };
+
+        const [width, height] = size.split('x').map(Number);
+        return { width, height };
+    }
+
+    async listModels(_params?: ModelSearchPayload): Promise<AIModel[]> {
+        return [
+            {
+                id: "firefly-v3-text-to-image",
+                name: "Firefly v3 Text to Image",
+                provider: this.provider,
+                description: "Adobe Firefly v3 text to image generation model",
+                tags: ["image-generation"]
+            },
+            {
+                id: "firefly-v3-image-to-image", 
+                name: "Firefly v3 Image to Image",
+                provider: this.provider,
+                description: "Adobe Firefly v3 image to image generation model",
+                tags: ["image-generation"]
+            },
+            {
+                id: "firefly-v3-inpainting",
+                name: "Firefly v3 Inpainting",
+                provider: this.provider,
+                description: "Adobe Firefly v3 inpainting model",
+                tags: ["image-generation"]
+            }
+        ];
+    }
+
+    async validateConnection(): Promise<boolean> {
+        try {
+            const response = await fetch(`${this.endpoint}/auth/validate`, {
+                headers: {
+                    'x-api-key': this.options.apiKey
+                }
+            });
+            return response.ok;
+        } catch (error) {
+            this.logger.error("[Firefly] Connection validation failed", { error });
+            return false;
+        }
+    }
+
+    async generateEmbeddings(_options: EmbeddingsOptions): Promise<EmbeddingsResult> {
+        throw new Error("Embeddings not supported by Firefly");
+    }
+}

package/src/bedrock/converse.ts

@@ -0,0 +1,194 @@
+import { JSONSchema4 } from "json-schema";
+import { PromptSegment, PromptRole } from "@llumiverse/core";
+import {
+    ConversationRole,
+    ConverseRequest,
+    Message,
+    SystemContentBlock,
+    ContentBlock,
+} from "@aws-sdk/client-bedrock-runtime";
+
+function getJSONSafetyNotice(schema: JSONSchema4) {
+    return "The answer must be a JSON object using the following JSON Schema:\n" + JSON.stringify(schema, undefined, 2);
+}
+
+function roleConversion(role: PromptRole): ConversationRole {
+    return role === PromptRole.assistant ? ConversationRole.ASSISTANT : ConversationRole.USER;
+}
+
+function mimeToImageType(mime: string): "png" | "jpeg" | "gif" | "webp" {
+    if (mime.startsWith("image/")) {
+        return mime.split("/")[1] as "png" | "jpeg" | "gif" | "webp";
+    }
+    return "png";
+}
+
+function mimeToDocType(mime: string): "pdf" | "csv" | "doc" | "docx" | "xls" | "xlsx" | "html" | "txt" | "md" {
+    if (mime.startsWith("application/") || mime.startsWith("text/")) {
+        return mime.split("/")[1] as "pdf" | "csv" | "doc" | "docx" | "xls" | "xlsx" | "html" | "txt" | "md";
+    }
+    return "txt";
+}
+function mimeToVideoType(mime: string): "mov" | "mkv" | "mp4" | "webm" | "flv" | "mpeg" | "mpg" | "wmv" | "three_gp" {
+    if (mime.startsWith("video/")) {
+        return mime.split("/")[1] as "mov" | "mkv" | "mp4" | "webm" | "flv" | "mpeg" | "mpg" | "wmv" | "three_gp";
+    }
+    return "mp4";
+}
+
+async function readStreamAsString(stream: ReadableStream): Promise<string> {
+    const out: Buffer[] = [];
+    for await (const chunk of stream as any) {
+        out.push(Buffer.from(chunk));
+    }
+    return Buffer.concat(out).toString();
+}
+
+async function readStreamAsUint8Array(stream: ReadableStream): Promise<Uint8Array> {
+    const out: Buffer[] = [];
+    for await (const chunk of stream as any) {
+        out.push(Buffer.from(chunk));
+    }
+    return Buffer.concat(out);
+}
+
+export function converseConcatMessages(messages: Message[] | undefined): Message[] {
+    if (!messages) return [];
+    //Concatenate messages of the same role. Required to have alternative user and assistant roles
+    for (let i = 0; i < messages.length - 1; i++) {
+        if (messages[i].role === messages[i + 1].role) {
+            messages[i].content = messages[i].content?.concat(...(messages[i + 1].content || []));
+            messages.splice(i + 1, 1);
+            i--;
+        }
+    }
+    return messages;
+}
+
+export function converseSystemToMessages(system: SystemContentBlock[]): Message {
+    return {
+        content: [{text: system.map(system => system.text).join('\n').trim()}],
+        role: ConversationRole.USER
+    }; 
+}
+
+export function converseRemoveJSONprefill(messages: Message[] | undefined): Message[] {
+    //Remove the "```json" stop message
+    if (messages && messages.length > 0) {
+        if (messages[messages.length - 1].content?.[0].text === "```json") {
+            messages.pop();
+        }
+    }
+    return messages ?? [];
+}
+
+export async function fortmatConversePrompt(segments: PromptSegment[], schema?: JSONSchema4): Promise<ConverseRequest> {
+    //Non-const for concat
+    let system: SystemContentBlock[] = [];
+    const safety: SystemContentBlock[] = [];
+    let messages: Message[] = [];
+
+    for (const segment of segments) {
+        const parts: Message[] = [];
+
+        //File segments
+        if (segment.files)
+            for (const f of segment.files) {
+                const source = await f.getStream();
+                let content: ContentBlock[];
+
+                //Image file - "png" | "jpeg" | "gif" | "webp"
+                if (f.mime_type && f.mime_type.startsWith("image")) {
+                    content = [
+                        {
+                            image: {
+                                format: mimeToImageType(f.mime_type),
+                                source: { bytes: await readStreamAsUint8Array(source) },
+                            },
+                        },
+                    ];
+
+                    //Document file - "pdf | csv | doc | docx | xls | xlsx | html | txt | md"
+                } else if (f.mime_type && (f.mime_type.startsWith("text") || f.mime_type?.startsWith("application"))) {
+                    content = [
+                        { text: f.name },
+                        {
+                            document: {
+                                format: mimeToDocType(f.mime_type),
+                                name: f.name,
+                                source: { bytes: await readStreamAsUint8Array(source) },
+                            },
+                        },
+                    ];
+
+                    //Video file - "mov | mkv | mp4 | webm | flv | mpeg | mpg | wmv | three_gp"
+                } else if (f.mime_type && f.mime_type.startsWith("video")) {
+                    content = [
+                        {
+                            video: {
+                                format: mimeToVideoType(f.mime_type),
+                                source: { bytes: await readStreamAsUint8Array(source) },
+                            },
+                        },
+                    ];
+
+                    //Fallback, send string
+                } else {
+                    content = [{ text: await readStreamAsString(source) }];
+                }
+
+                parts.push({
+                    content: content,
+                    role: roleConversion(segment.role),
+                });
+            }
+
+        //Text segments
+        if (segment.content) {
+            parts.push({
+                content: [{ text: segment.content }],
+                role: roleConversion(segment.role),
+            });
+        }
+
+        if (segment.role === PromptRole.system) {
+            system.push({ text: segment.content });
+        } else if (segment.role === PromptRole.safety) {
+            safety.push({ text: segment.content });
+        } else if (segment.role !== PromptRole.negative && segment.role !== PromptRole.mask) {
+            //User or Assistant
+            messages = messages.concat(parts);
+        }
+    }
+
+    //Conversations must start with a user message
+    //Use the system messages if none are provided
+    if (messages.length === 0) {
+        const systemMessage = converseSystemToMessages(system);
+        if (systemMessage?.content && systemMessage.content[0].text) {
+            messages.push(systemMessage);
+        } else {
+            throw new Error("Prompt must contain at least one message");
+        }
+        system = [];
+    }
+
+    if (schema) {
+        safety.push({ text: "IMPORTANT: " + getJSONSafetyNotice(schema) });
+        system = system.concat(safety);
+
+        //prefill the json
+        messages.push({
+            content: [{ text: "```json" }],
+            role: ConversationRole.ASSISTANT,
+        });
+    }
+
+    messages = converseConcatMessages(messages);
+
+    return {
+        modelId: undefined, //required property, but allowed to be undefined
+        messages: messages,
+        system: system,
+    };
+}