auricui-skill 0.0.0-stage → 0.1.2

package/skill/assets/upstream/apps/desktop/chat/src/lib/chat-store.ts

@@ -0,0 +1,101 @@
+/**
+ * A project's chat record, kept by the app in the project's own folder (`.film/chat`, src/chat-store.mjs): its
+ * conversations, the one shown last, and each finished turn with what the chat shows of it. Read once when the
+ * project opens (openProjectChat), held here so the chat reads it as it always has, and written back as it changes.
+ */
+import React from 'react';
+import type { Turn } from '@openfilm/shared';
+import type { TranscriptItem } from '@/lib/agent-transcript';
+import type { ProjectSessionItem } from '@/lib/project-sessions';
+import { app } from '../app-bridge';
+
+export interface SavedTurn { turn: Turn; items: TranscriptItem[] }
+interface ProjectChat { sessions: ProjectSessionItem[]; last: string | null; turns: SavedTurn[] }
+
+const projects = new Map<string, ProjectChat>();
+const loading = new Map<string, Promise<void>>();
+/** which project each folder's record was read for: a project deleted and another made in its folder (a new
+ * "Untitled" where the last one was) is not the same project, and does not get its chat */
+const readFor = new Map<string, string>();
+const EMPTY: ProjectChat = { sessions: [], last: null, turns: [] };
+
+/**
+ * Read the project's record from its folder (once a launch, and again when another project takes the folder); the
+ * chat waits for it before it shows the project. `project`: its folder; `id`: Studio's id for it.
+ */
+export function openProjectChat(project: string, id: string): Promise<void> {
+  if (readFor.get(project) !== id) {
+    readFor.set(project, id);
+    projects.delete(project);
+  }
+  if (projects.has(project)) return Promise.resolve();
+  const key = `${id}\0${project}`;
+  const inFlight = loading.get(key);
+  if (inFlight) return inFlight;
+  const mine = () => readFor.get(project) === id;
+  const pending: Promise<void> = app.chat.load(project)
+    .then((chat) => {
+      if (!mine()) return;
+      projects.set(project, {
+        sessions: Array.isArray(chat?.sessions) ? chat.sessions as ProjectSessionItem[] : [],
+        last: typeof chat?.last === 'string' ? chat.last : null,
+        turns: Array.isArray(chat?.turns) ? chat.turns as SavedTurn[] : [],
+      });
+    })
+    /* not readable (not one of Studio's projects yet, a folder gone): an empty record, written once it changes */
+    .catch(() => { if (mine()) projects.set(project, { ...EMPTY, sessions: [], turns: [] }); })
+    .finally(() => { loading.delete(key); });
+  loading.set(key, pending);
+  return pending;
+}
+
+const record = (project: string): ProjectChat => {
+  let chat = projects.get(project);
+  if (!chat) { chat = { ...EMPTY, sessions: [], turns: [] }; projects.set(project, chat); }
+  return chat;
+};
+
+/* the conversations are written a moment after the last change: renaming one, or a turn touching it, is a burst */
+const sessionWrites = new Map<string, ReturnType<typeof setTimeout>>();
+function writeSessions(project: string) {
+  clearTimeout(sessionWrites.get(project));
+  sessionWrites.set(project, setTimeout(() => {
+    sessionWrites.delete(project);
+    const chat = record(project);
+    void app.chat.saveSessions(project, chat.sessions, chat.last).catch(() => { /* written next time */ });
+  }, 150));
+}
+
+export const chatSessions = (project: string): ProjectSessionItem[] => record(project).sessions;
+export function setChatSessions(project: string, sessions: ProjectSessionItem[]) {
+  record(project).sessions = sessions;
+  writeSessions(project);
+}
+
+export const lastSession = (project: string): string | null => record(project).last;
+export function setLastSession(project: string, id: string) {
+  const chat = record(project);
+  if (chat.last === id) return;
+  chat.last = id;
+  writeSessions(project);
+}
+
+export const chatTurns = (project: string): SavedTurn[] => record(project).turns;
+export function putChatTurn(project: string, entry: SavedTurn) {
+  const chat = record(project);
+  chat.turns = [...chat.turns.filter((t) => t.turn.id !== entry.turn.id), entry];
+  void app.chat.saveTurn(project, entry.turn, entry.items).catch(() => { /* kept for this launch */ });
+}
+
+/** Whether `project`'s record has been read (reading it if not): the chat shows the project once it has. */
+export function useProjectChat(project: { id: string; path: string } | null): boolean {
+  const key = project ? `${project.id}\0${project.path}` : null;
+  const [loaded, setLoaded] = React.useState<string | null>(() => (project && readFor.get(project.path) === project.id && projects.has(project.path) ? key : null));
+  React.useEffect(() => {
+    if (!project || !key) return undefined;
+    let alive = true;
+    void openProjectChat(project.path, project.id).then(() => { if (alive) setLoaded(key); });
+    return () => { alive = false; };
+  }, [key]);
+  return key != null && loaded === key;
+}

package/skill/assets/upstream/apps/desktop/chat/src/lib/composer-attachments.ts

@@ -0,0 +1,189 @@
+/**
+ * Putting things into the composer: dropped, pasted or picked from "+", all take this path.
+ *
+ * Two kinds: images go into the first message as references; everything else goes the files way, as a path only.
+ * Sorting turns nothing away, for format, count or size.
+ *
+ * Sorting is kept apart from reading bytes: the first is pure and tested with plain objects; the second only
+ * makes sense in a browser (FileReader) and has no branches worth testing.
+ */
+import { TURN_ATTACHMENT_THUMB_MAX_CHARS } from '@openfilm/shared';
+import {
+  DOCUMENT_MEDIA_TYPES,
+  IMAGE_EXTENSIONS,
+  VIDEO_UPLOAD_EXTENSIONS,
+} from './upload-limits';
+
+/** A batch of files sorted in two. Nothing is refused: not for format, count or size. */
+export interface AttachmentPlan<T> {
+  images: T[];
+  documents: T[];
+}
+
+/** The file name's extension (lower case, no dot); empty when there is no dot. */
+function extensionOf(name: string): string {
+  const dot = name.lastIndexOf('.');
+  return dot > 0 ? name.slice(dot + 1).toLowerCase() : '';
+}
+
+/**
+ * Whether this is an image.
+ *
+ * MIME first: a pasted screenshot often has no useful name (Chrome says "image.png", Safari sometimes nothing),
+ * so the extension tells nothing. Only without a MIME type does the extension decide, as for some files dragged
+ * from Finder.
+ */
+export function isImageAttachment(file: { name: string; type: string }): boolean {
+  if (file.type.startsWith('image/')) return true;
+  return (IMAGE_EXTENSIONS as readonly string[]).includes(extensionOf(file.name));
+}
+
+/**
+ * Whether this file goes as a path: everything but images.
+ *
+ * The model never reads a file's bytes; pdf or mp4, it gets a path, and a path works for any format. So this asks
+ * only "is it an image", not "what format is it".
+ */
+export function isDocumentAttachment(file: { name: string; type: string }): boolean {
+  return !isImageAttachment(file);
+}
+
+/** Whether this is a video: MIME first, the extension only without one (as in isImageAttachment). */
+export function isVideoAttachment(file: { name: string; type?: string; mediaType?: string }): boolean {
+  const mime = file.type || file.mediaType || '';
+  if (mime.startsWith('video/')) return true;
+  return VIDEO_UPLOAD_EXTENSIONS.has(extensionOf(file.name));
+}
+
+/** The mediaType sent on: from the table, else the one the browser reports. */
+export function documentMediaType(file: { name: string; type: string }): string {
+  return DOCUMENT_MEDIA_TYPES[extensionOf(file.name)] ?? file.type ?? 'application/octet-stream';
+}
+
+/**
+ * Splits the batch into images and paths, in the order the person attached them. None is turned away.
+ *
+ * Generic only so tests can pass plain { name, type } objects; callers pass File[].
+ */
+export function planComposerAttachments<T extends { name: string; type: string }>(
+  files: readonly T[],
+): AttachmentPlan<T> {
+  const images: T[] = [];
+  const documents: T[] = [];
+  for (const file of files) {
+    if (isImageAttachment(file)) images.push(file);
+    else documents.push(file);
+  }
+  return { images, documents };
+}
+
+/**
+ * The files in a paste or a drop.
+ *
+ * `files` is the standard place, but Safari sometimes pastes into `items` only and leaves `files` empty, so both
+ * are read.
+ */
+export function filesFromTransfer(data: DataTransfer | null | undefined): File[] {
+  if (!data) return [];
+  if (data.files && data.files.length > 0) return Array.from(data.files);
+  return Array.from(data.items ?? [])
+    .filter((item) => item.kind === 'file')
+    .map((item) => item.getAsFile())
+    .filter((file): file is File => file !== null);
+}
+
+/** Whether this drop or paste carries files from the system (not text, nor a drag inside the app). */
+export function transferHasFiles(data: DataTransfer | null | undefined): boolean {
+  return !!data && Array.from(data.types).includes('Files');
+}
+
+/**
+ * Thumbnail edge sizes, tried largest first.
+ *
+ * 320 so it still reads when opened: this copy is not only the 40px tile in the message but the only thing the
+ * person can open later; the original goes into the agent's workspace and never comes back to the UI.
+ *
+ * Over the limit, the next size is tried rather than giving up: a busy photo may not fit at 320 but is still
+ * recognizable at 224, which is all this copy is for.
+ */
+const THUMB_STEPS = [320, 224, 160] as const;
+
+/**
+ * A thumbnail that stays in the conversation.
+ *
+ * The original image goes to the agent's workspace and never comes back to the UI; without a copy, the person
+ * cannot tell later which image they sent. So a copy of some tens of KB is kept with the message.
+ *
+ * webp, not jpeg: screenshots are often UI with transparent backgrounds, which jpeg turns black. Without webp
+ * support toDataURL falls back to png, which is larger, hence the sizes.
+ * Any failure returns undefined: a missing thumbnail must not block a message.
+ */
+export function makeAttachmentThumb(dataUrl: string): Promise<string | undefined> {
+  return new Promise((resolve) => {
+    const image = new Image();
+    image.onload = () => {
+      try {
+        const canvas = document.createElement('canvas');
+        const ctx = canvas.getContext('2d');
+        if (!ctx) return resolve(undefined);
+        for (const edge of THUMB_STEPS) {
+          const longest = Math.max(image.naturalWidth, image.naturalHeight);
+          // An SVG may have no intrinsic size (naturalWidth 0): fill the edge, don't divide by zero.
+          const scale = longest > 0 ? Math.min(1, edge / longest) : 1;
+          canvas.width = Math.max(1, Math.round((image.naturalWidth || edge) * scale));
+          canvas.height = Math.max(1, Math.round((image.naturalHeight || edge) * scale));
+          ctx.clearRect(0, 0, canvas.width, canvas.height);
+          ctx.drawImage(image, 0, 0, canvas.width, canvas.height);
+          const thumb = canvas.toDataURL('image/webp', 0.75);
+          if (thumb.length <= TURN_ATTACHMENT_THUMB_MAX_CHARS) return resolve(thumb);
+        }
+        resolve(undefined);
+      } catch {
+        // A cross-origin image taints the canvas and toDataURL throws; inputs here are data URLs, but still.
+        resolve(undefined);
+      }
+    };
+    image.onerror = () => resolve(undefined);
+    image.src = dataUrl;
+  });
+}
+
+/**
+ * One frame of a video as a data URL, the thumbnail for its tile.
+ *
+ * At 1 s (a tenth in for short clips), not frame 0: many clips fade in from black.
+ * Formats the browser can't decode (often mkv, avi) end in onerror, and the caller keeps the file-name tile.
+ */
+export function videoFrameDataUrl(file: Blob): Promise<string> {
+  return new Promise((resolve, reject) => {
+    const url = URL.createObjectURL(file);
+    const video = document.createElement('video');
+    const end = (frame?: string, error?: unknown) => {
+      video.removeAttribute('src'); video.load(); URL.revokeObjectURL(url);
+      if (frame) resolve(frame); else reject(error ?? new Error('no frame'));
+    };
+    video.muted = true;
+    video.preload = 'auto';
+    video.onloadedmetadata = () => { video.currentTime = Math.min(1, (video.duration || 0) * 0.1); };
+    video.onseeked = () => {
+      try {
+        const canvas = document.createElement('canvas');
+        canvas.width = video.videoWidth;
+        canvas.height = video.videoHeight;
+        canvas.getContext('2d')?.drawImage(video, 0, 0);
+        end(canvas.toDataURL('image/jpeg', 0.85));
+      } catch (error) { end(undefined, error); }
+    };
+    video.onerror = () => end(undefined, video.error);
+    video.src = url;
+  });
+}
+
+export function readFileDataUrl(file: Blob): Promise<string> {
+  return new Promise((resolve, reject) => {
+    const reader = new FileReader();
+    reader.onload = () => resolve(String(reader.result));
+    reader.onerror = () => reject(reader.error ?? new Error('read failed'));
+    reader.readAsDataURL(file);
+  });
+}

package/skill/assets/upstream/apps/desktop/chat/src/lib/desktop-attachments.ts

@@ -0,0 +1,178 @@
+/**
+ * The files brought into a message (dragged in, pasted, picked with +): a row of thumbnails (a picture's own, a
+ * video's frame) and names above the sentence. Any number, and folders too. A file or folder from the disk stays where
+ * it is: the message names its path, as an agent's own chat does, and the agent copies into the project what the film
+ * uses. Only a pasted one, which has no path, is kept in the project (.film/attachments/). The message says where they
+ * are (attachedText), the bubble shows their names and thumbnails. Pictures are also sent to the agent beside the text,
+ * as one pasted in its own chat is, to an agent that takes pictures (attachedPictures).
+ */
+import React from 'react';
+import type { TurnAttachment } from '@openfilm/shared';
+import { isImageAttachment, isVideoAttachment, makeAttachmentThumb, readFileDataUrl, videoFrameDataUrl } from '@/lib/composer-attachments';
+import { attachmentKind } from '@/lib/project-resources';
+import { app } from '../app-bridge';
+import type { PromptImage } from '@/lib/desktop-bridge';
+import { attachedPicture } from '@/lib/ref-pictures';
+
+export interface DesktopAttachment {
+  id: string;
+  name: string;
+  kind: 'image' | 'document';
+  mediaType: string;
+  size: number;
+  /** a folder from the disk: its path names it, nothing in it is read */
+  folder?: boolean;
+  /** the file itself, while it is in the composer: an image shows it, and any one opens it to look at (AttachmentPreview) */
+  previewUrl?: string;
+  /** the small picture kept with the message: an image's, or a frame of a video */
+  thumb?: string;
+  state: 'pending' | 'ready' | 'failed';
+  /** where it is: its own path on the disk, or where a pasted one was kept in the project */
+  path?: string;
+  /** the file, until the message goes: a picture is read for its agent only then, and only as many as are sent */
+  file?: File;
+  error?: string;
+}
+
+/** at most this many pictures go with a message (the app sends no more: src/local-agents-ipc.mjs PROMPT_IMAGES_MAX) */
+export const PROMPT_PICTURES_MAX = 20;
+
+/** thumbnails made at once: a folder's worth of photos read all together would hold every one of them in memory */
+const THUMBS_AT_ONCE = 4;
+
+const SIZE_UNITS = ['B', 'KB', 'MB', 'GB'];
+export function formatFileSize(bytes: number): string {
+  let value = bytes;
+  let unit = 0;
+  while (value >= 1024 && unit < SIZE_UNITS.length - 1) { value /= 1024; unit += 1; }
+  return `${unit ? value.toFixed(value < 10 ? 1 : 0) : value} ${SIZE_UNITS[unit]}`;
+}
+
+export function useDesktopAttachments(project: string) {
+  const [files, setFiles] = React.useState<DesktopAttachment[]>([]);
+  const patch = (id: string, next: Partial<DesktopAttachment>) => setFiles((list) => list.map((f) => (f.id === id ? { ...f, ...next } : f)));
+
+  const thumbs = React.useRef<{ running: number; waiting: Array<() => Promise<void>> }>({ running: 0, waiting: [] });
+  const thumbnail = React.useCallback((make: () => Promise<void>) => {
+    const queue = thumbs.current;
+    const next = () => {
+      if (queue.running >= THUMBS_AT_ONCE) return;
+      const job = queue.waiting.shift();
+      if (!job) return;
+      queue.running += 1;
+      void job().catch(() => {}).finally(() => { queue.running -= 1; next(); });
+    };
+    queue.waiting.push(make);
+    next();
+  }, []);
+
+  const add = React.useCallback((incoming: File[]) => {
+    const bridge = (window as unknown as { openfilmDesktop?: { pathForFile?: (f: File) => string } }).openfilmDesktop;
+    const items = incoming.map((file) => {
+      const image = isImageAttachment(file);
+      const item: DesktopAttachment = {
+        id: `a${Date.now().toString(36)}${Math.random().toString(36).slice(2, 8)}`,
+        name: file.name || (image ? 'image.png' : 'file'),
+        kind: image ? 'image' : 'document',
+        mediaType: file.type,
+        size: file.size,
+        previewUrl: URL.createObjectURL(file),
+        state: 'pending',
+        file,
+      };
+      return { item, file, path: bridge?.pathForFile?.(file) ?? '' };
+    });
+    setFiles((list) => [...list, ...items.map(({ item }) => item)]);
+    for (const { item, file } of items) {
+      if (item.kind === 'image') {
+        thumbnail(() => readFileDataUrl(file).then(makeAttachmentThumb).then((thumb) => { if (thumb) patch(item.id, { thumb }); }));
+      } else if (isVideoAttachment(file)) {
+        /* a video shows a frame of itself, as a picture does: its name alone says little of what was sent */
+        thumbnail(() => videoFrameDataUrl(file).then(makeAttachmentThumb).then((thumb) => { if (thumb) patch(item.id, { thumb }); }));
+      }
+    }
+    /* from the disk: its path, once the app has said whether it is a file or a folder (nothing is copied) */
+    const onDisk = items.filter(({ path }) => path);
+    if (onDisk.length) {
+      void app.chat.describePaths(onDisk.map(({ path }) => path))
+        .then((found) => onDisk.forEach(({ item }, i) => {
+          const one = found[i];
+          if (!one || 'error' in one) { patch(item.id, { state: 'failed', error: one?.error ?? 'That is not a file.' }); return; }
+          if (one.folder) {
+            URL.revokeObjectURL(item.previewUrl!);
+            patch(item.id, { state: 'ready', path: one.path, folder: true, kind: 'document', previewUrl: undefined, file: undefined, thumb: undefined });
+          } else patch(item.id, { state: 'ready', path: one.path, size: one.size ?? item.size });
+        }))
+        .catch((error: unknown) => onDisk.forEach(({ item }) => patch(item.id, { state: 'failed', error: error instanceof Error ? error.message : String(error) })));
+    }
+    /* pasted: no path of its own, so its bytes are kept in the project */
+    for (const { item, file, path } of items) {
+      if (path) continue;
+      void file.arrayBuffer()
+        .then((bytes) => app.chat.saveAttachment(project, item.name, bytes))
+        .then((r) => patch(item.id, { state: 'ready', path: r.path }))
+        .catch((error: unknown) => patch(item.id, { state: 'failed', error: error instanceof Error ? error.message : String(error) }));
+    }
+  }, [project, thumbnail]);
+
+  const drop = React.useCallback((id: string) => {
+    setFiles((list) => {
+      const gone = list.find((f) => f.id === id);
+      if (gone?.previewUrl) URL.revokeObjectURL(gone.previewUrl);
+      return list.filter((f) => f.id !== id);
+    });
+  }, []);
+
+  /** Hand over the files for sending; the composer is empty of them after. */
+  const current = React.useRef(files);
+  current.current = files;
+  const take = React.useCallback((): DesktopAttachment[] => {
+    const taken = current.current;
+    /* sent: the message keeps their thumbnails, the agent their paths */
+    for (const f of taken) if (f.previewUrl) URL.revokeObjectURL(f.previewUrl);
+    current.current = [];
+    setFiles([]);
+    return taken;
+  }, []);
+
+  return { files, add, drop, take };
+}
+
+/** What still keeps the message from going: a file still on its way, or one that could not be attached. */
+export function attachmentBlocker(files: readonly DesktopAttachment[], t: (path: string) => string): string | null {
+  if (files.some((f) => f.state === 'pending')) return t('composer.filesPending');
+  const failed = files.filter((f) => f.state === 'failed');
+  return failed.length
+    ? t('composer.filesFailed').replace('{names}', failed.map((f) => f.name).join(', ')).replace('{error}', failed[0]!.error ?? '')
+    : null;
+}
+
+/** After the message, for the agent: where the files are. A folder's path ends in "/". */
+export function attachedText(files: readonly DesktopAttachment[]): string {
+  const placed = files.filter((f) => f.path);
+  if (!placed.length) return '';
+  const line = (f: DesktopAttachment) => (f.folder
+    ? `- ${f.path!.replace(/[\\/]+$/, '')}/ (folder)`
+    : `- ${f.path} (${attachmentKind({ name: f.name, mediaType: f.mediaType })}, ${formatFileSize(f.size)})`);
+  return `\n\nAttached files:\n${placed.map(line).join('\n')}`;
+}
+
+/**
+ * The pictures among them, as their agent is sent them: the first `room`, in the order they were attached (the rest go
+ * by their path only). Read now, as the message goes: a folder's worth of photos would not all be sent anyway.
+ */
+export async function attachedPictures(files: readonly DesktopAttachment[], room: number): Promise<PromptImage[]> {
+  const wanted = files.filter((f) => f.kind === 'image' && f.file).slice(0, Math.max(0, room));
+  const made = await Promise.all(wanted.map((f) => attachedPicture(f.file!)));
+  return made.filter((picture): picture is PromptImage => picture !== null);
+}
+
+/** What the message keeps of them, to show in its bubble (a folder's name ends in "/", and its path opens it). */
+export function turnAttachments(files: readonly DesktopAttachment[]): TurnAttachment[] {
+  return files.map((f) => ({
+    kind: f.kind,
+    name: f.folder ? `${f.name}/` : f.name,
+    ...(f.thumb ? { thumb: f.thumb } : {}),
+    ...(f.folder && f.path ? { path: f.path } : {}),
+  }));
+}