@sanity/assist 1.0.12 → 1.1.0

package/README.md

@@ -200,6 +200,33 @@ but some common caveats to the field that you may run into using AI Assist are:
 * Timeouts: To be able to write structured content, we're using the largest language models - long-running results may time out or intermittently fail
 * Limited capacity: The underlying LLM APIs used by AI Assist are resource constrained
 
+## Other features
+
+### Caption generation
+AI Assist can optionally generate captions for images. This has to be enabled on an image-type/field,
+by setting the `options.captionField` on the image type, where `captionField` is the field name of a 
+custom string-field on the image object:
+
+```tsx
+defineField({
+    type: 'image',
+    name: 'inlineImage',
+    title: 'Image',
+    fields: [
+      defineField({
+        type: 'string',
+        name: 'caption',
+        title: 'Caption',
+      }),
+    ],
+    options: {
+      captionField: 'caption', 
+    },
+}),
+```
+This will add a "Generate caption" action to the configured field. 
+"Generate caption" action will automatically run whenever the image changes.
+
 ## License
 
 [MIT](LICENSE) © Sanity

package/dist/index.d.ts

@@ -40,6 +40,7 @@ declare module 'sanity' {
   interface GeopointOptions extends AssistOptions {}
   interface ImageOptions extends AssistOptions {
     imagePromptField?: string
+    captionField?: string
   }
   interface NumberOptions extends AssistOptions {}
   interface ObjectOptions extends AssistOptions {}

package/dist/index.esm.js

@@ -120,6 +120,20 @@ function isType(schemaType, typeName) {
   }
   return isType(schemaType.type, typeName);
 }
+function isImage(schemaType) {
+  return isType(schemaType, "image");
+}
+function getCaptionFieldOption(schemaType) {
+  var _a;
+  if (!schemaType) {
+    return void 0;
+  }
+  const captionField = (_a = schemaType.options) == null ? void 0 : _a.captionField;
+  if (captionField) {
+    return captionField;
+  }
+  return getCaptionFieldOption(schemaType.type);
+}
 function isSchemaAssistEnabled(type) {
   var _a, _b;
   return !((_b = (_a = type.options) == null ? void 0 : _a.aiWritingAssistance) == null ? void 0 : _b.exclude);
@@ -736,10 +750,11 @@ function InstructionTaskHistoryButton(props) {
   const titledTasks = useMemo(() => {
     var _a2;
     const t = (_a2 = tasks == null ? void 0 : tasks.filter(task => task.started && /* @__PURE__ */new Date().getTime() - new Date(task.started).getTime() < maxHistoryVisibilityMs).map(task => {
+      var _a3;
       const instruction = instructions == null ? void 0 : instructions.find(i => i._key === task.instructionKey);
       return {
         ...task,
-        title: showTitles ? getInstructionTitle(instruction) : void 0,
+        title: showTitles ? (_a3 = task.title) != null ? _a3 : getInstructionTitle(instruction) : void 0,
         cancel: () => cancelRun(task._key)
       };
     })) != null ? _a2 : [];
@@ -893,6 +908,48 @@ function useApiClient(customApiClient) {
   });
   return useMemo(() => customApiClient ? customApiClient(client) : client, [client, customApiClient]);
 }
+function useGenerateCaption(apiClient) {
+  const [loading, setLoading] = useState(false);
+  const user = useCurrentUser();
+  const schema = useSchema();
+  const types = useMemo(() => serializeSchema(schema, {
+    leanFormat: true
+  }), [schema]);
+  const toast = useToast();
+  const generateCaption = useCallback(_ref4 => {
+    let {
+      path,
+      documentId
+    } = _ref4;
+    setLoading(true);
+    return apiClient.request({
+      method: "POST",
+      url: "/assist/tasks/generate-caption/".concat(apiClient.config().dataset, "?projectId=").concat(apiClient.config().projectId),
+      body: {
+        path,
+        documentId,
+        types,
+        userId: user == null ? void 0 : user.id
+      }
+    }).catch(e => {
+      toast.push({
+        status: "error",
+        title: "Generate caption failed",
+        description: e.message
+      });
+      setLoading(false);
+      throw e;
+    }).finally(() => {
+      setTimeout(() => {
+        setLoading(false);
+      }, 2e3);
+    });
+  }, [setLoading, apiClient, toast, user, types]);
+  return useMemo(() => ({
+    generateCaption,
+    loading
+  }), [generateCaption, loading]);
+}
 function useGetInstructStatus(apiClient) {
   const [loading, setLoading] = useState(true);
   const projectClient = useClient({
@@ -1093,8 +1150,8 @@ function RunInstructionProvider(props) {
       runInstructionRequest({
         ...request,
         instructionKey: instruction._key,
-        userTexts: Object.entries(inputs).map(_ref4 => {
-          let [key, value] = _ref4;
+        userTexts: Object.entries(inputs).map(_ref5 => {
+          let [key, value] = _ref5;
           return {
             blockKey: key,
             userInput: value
@@ -1107,8 +1164,8 @@ function RunInstructionProvider(props) {
   const open = !!runRequest;
   const runDisabled = useMemo(() => {
     var _a2, _b;
-    return ((_b = (_a2 = runRequest == null ? void 0 : runRequest.userInputBlocks) == null ? void 0 : _a2.length) != null ? _b : 0) > Object.entries(inputs).filter(_ref5 => {
-      let [, value] = _ref5;
+    return ((_b = (_a2 = runRequest == null ? void 0 : runRequest.userInputBlocks) == null ? void 0 : _a2.length) != null ? _b : 0) > Object.entries(inputs).filter(_ref6 => {
+      let [, value] = _ref6;
       return !!value;
     }).length;
   }, [runRequest == null ? void 0 : runRequest.userInputBlocks, inputs]);
@@ -1688,10 +1745,10 @@ const assistInspector = {
     showAsAction: false
   }),
   component: AssistInspectorWrapper,
-  onClose(_ref6) {
+  onClose(_ref7) {
     let {
       params
-    } = _ref6;
+    } = _ref7;
     return {
       params: typed({
         ...params,
@@ -1764,18 +1821,19 @@ var __template$3 = (cooked, raw) => __freeze$3(__defProp$3(cooked, "raw", {
 var _a$3, _b$1;
 const fadeIn = keyframes(_a$3 || (_a$3 = __template$3(["\n  0% {\n    opacity: 0;\n    transform: scale(0.75);\n  }\n  40% {\n    opacity: 0;\n    transform: scale(0.75);\n  }\n  100% {\n    opacity: 1;\n    transform: scale(1);\n  }\n"])));
 const FadeInDiv = styled.div(_b$1 || (_b$1 = __template$3(["\n  animation-name: ", ";\n  animation-timing-function: ease-in-out;\n"])), fadeIn);
-function FadeInContent(_ref7) {
+const FadeInContent = forwardRef(function FadeInContent2(_ref8, ref) {
   let {
     children,
     durationMs = 250
-  } = _ref7;
+  } = _ref8;
   return /* @__PURE__ */jsx(FadeInDiv, {
+    ref,
     style: {
       animationDuration: "".concat(durationMs, "ms")
     },
     children
   });
-}
+});
 const purple = {
   "50": {
     title: "Purple 50",
@@ -3062,13 +3120,13 @@ function useSelectedSchema(fieldPath, documentSchema) {
     return currentSchema;
   }, [documentSchema, fieldPath]);
 }
-function FieldsInitializer(_ref8) {
+function FieldsInitializer(_ref9) {
   let {
     pathKey,
     activePath,
     fieldExists,
     onChange
-  } = _ref8;
+  } = _ref9;
   const initialized = useRef(false);
   useEffect(() => {
     if (initialized.current || fieldExists || activePath || !pathKey) {
@@ -3240,8 +3298,8 @@ function IconInput(props) {
     onChange
   } = props;
   const id = useId();
-  const items = useMemo(() => Object.entries(icons).map(_ref9 => {
-    let [key, icon] = _ref9;
+  const items = useMemo(() => Object.entries(icons).map(_ref10 => {
+    let [key, icon] = _ref10;
     return /* @__PURE__ */jsx(IconItem, {
       iconKey: key,
       icon,
@@ -3269,12 +3327,12 @@ function IconInput(props) {
     }
   });
 }
-function IconItem(_ref10) {
+function IconItem(_ref11) {
   let {
     icon,
     iconKey: key,
     onChange
-  } = _ref10;
+  } = _ref11;
   const onClick = useCallback(() => onChange(set(key)), [onChange, key]);
   return /* @__PURE__ */jsx(MenuItem, {
     icon,
@@ -3285,8 +3343,8 @@ function IconItem(_ref10) {
 }
 function getIcon(iconName) {
   var _a, _b;
-  return (_b = (_a = Object.entries(icons).find(_ref11 => {
-    let [key] = _ref11;
+  return (_b = (_a = Object.entries(icons).find(_ref12 => {
+    let [key] = _ref12;
     return key === iconName;
   })) == null ? void 0 : _a[1]) != null ? _b : icons.sparkles;
 }
@@ -3540,11 +3598,11 @@ const contextDocumentSchema = defineType({
       title: "title",
       context: "context"
     },
-    prepare(_ref12) {
+    prepare(_ref13) {
       let {
         title,
         context
-      } = _ref12;
+      } = _ref13;
       var _a;
       const text = context == null ? void 0 : context.flatMap(block => block == null ? void 0 : block.children).flatMap(child => {
         var _a2;
@@ -3673,10 +3731,10 @@ const fieldReference = defineType({
     select: {
       path: "path"
     },
-    prepare(_ref13) {
+    prepare(_ref14) {
       let {
         path
-      } = _ref13;
+      } = _ref14;
       return {
         title: path,
         path,
@@ -3822,12 +3880,12 @@ const instruction = defineType({
       title: "title",
       userId: "userId"
     },
-    prepare: _ref14 => {
+    prepare: _ref15 => {
       let {
         icon,
         title,
         userId
-      } = _ref14;
+      } = _ref15;
       return {
         title,
         icon: icon ? icons[icon] : SparklesIcon,
@@ -4115,6 +4173,100 @@ function PrivateIcon() {
     children: /* @__PURE__ */jsx(LockIcon, {})
   });
 }
+const ImageContext = createContext(void 0);
+function ImageContextProvider(props) {
+  var _a;
+  const {
+    schemaType,
+    path,
+    value
+  } = props;
+  const assetRef = (_a = value == null ? void 0 : value.asset) == null ? void 0 : _a._ref;
+  const [assetRefState, setAssetRefState] = useState(assetRef);
+  const {
+    documentId
+  } = useAssistDocumentContext();
+  const {
+    config
+  } = useAiAssistanceConfig();
+  const apiClient = useApiClient(config == null ? void 0 : config.__customApiClient);
+  const {
+    generateCaption
+  } = useGenerateCaption(apiClient);
+  useEffect(() => {
+    const captionField = getCaptionFieldOption(schemaType);
+    if (assetRef && documentId && captionField && assetRef !== assetRefState) {
+      setAssetRefState(assetRef);
+      generateCaption({
+        path: pathToString([...path, captionField]),
+        documentId
+      });
+    }
+  }, [schemaType, path, assetRef, assetRefState, documentId, generateCaption]);
+  const context = useMemo(() => {
+    const captionField = getCaptionFieldOption(schemaType);
+    return captionField ? {
+      captionPath: pathToString([...path, captionField]),
+      assetRef
+    } : void 0;
+  }, [schemaType, path, assetRef]);
+  return /* @__PURE__ */jsx(ImageContext.Provider, {
+    value: context,
+    children: props.renderDefault(props)
+  });
+}
+function node$1(node2) {
+  return node2;
+}
+const generateCaptionsActions = {
+  name: "sanity-assist-generate-captions",
+  useAction(props) {
+    const pathKey = usePathKey(props.path);
+    const {
+      config
+    } = useAiAssistanceConfig();
+    const apiClient = useApiClient(config == null ? void 0 : config.__customApiClient);
+    const {
+      generateCaption,
+      loading
+    } = useGenerateCaption(apiClient);
+    const imageContext = useContext(ImageContext);
+    if (imageContext && pathKey === (imageContext == null ? void 0 : imageContext.captionPath)) {
+      const {
+        documentId
+      } = useAssistDocumentContext();
+      return useMemo(() => {
+        return node$1({
+          type: "action",
+          icon: loading ? () => /* @__PURE__ */jsx(Box, {
+            style: {
+              height: 17
+            },
+            children: /* @__PURE__ */jsx(Spinner, {
+              style: {
+                transform: "translateY(6px)"
+              }
+            })
+          }) : ImageIcon,
+          title: "Generate caption",
+          onAction: () => {
+            if (loading) {
+              return;
+            }
+            generateCaption({
+              path: pathKey,
+              documentId: documentId != null ? documentId : ""
+            });
+          },
+          renderAsButton: true,
+          disabled: loading,
+          hidden: !imageContext.assetRef
+        });
+      }, [generateCaption, pathKey, documentId, loading, imageContext]);
+    }
+    return void 0;
+  }
+};
 function node(node2) {
   return node2;
 }
@@ -4164,6 +4316,7 @@ const assistFieldActions = {
     const isInspectorOpen = (inspector == null ? void 0 : inspector.name) === aiInspectorId;
     const isPathSelected = pathKey === selectedPath;
     const isSelected = isInspectorOpen && isPathSelected;
+    const imageCaptionAction = generateCaptionsActions.useAction(props);
     const manageInstructions = useCallback(() => isSelected ? closeInspector(aiInspectorId) : openInspector(aiInspectorId, {
       [fieldPathParam]: pathKey,
       [instructionParam]: void 0
@@ -4193,17 +4346,17 @@ const assistFieldActions = {
         type: "group",
         icon: () => null,
         title: "Run instructions",
-        children: instructions.map(instruction => instructionItem({
+        children: [...instructions.map(instruction => instructionItem({
           instruction,
           isPrivate: Boolean(instruction.userId && instruction.userId === (currentUser == null ? void 0 : currentUser.id)),
           onInstructionAction,
           hidden: isHidden,
           documentIsNew: !!documentIsNew,
           assistSupported
-        })),
+        })), imageCaptionAction].filter(Boolean),
         expanded: true
       }) : void 0;
-    }, [instructions, currentUser == null ? void 0 : currentUser.id, onInstructionAction, isHidden, documentIsNew, assistSupported]);
+    }, [instructions, currentUser == null ? void 0 : currentUser.id, onInstructionAction, isHidden, documentIsNew, assistSupported, imageCaptionAction]);
     const instructionsLength = (instructions == null ? void 0 : instructions.length) || 0;
     const manageInstructionsItem = useMemo(() => node({
       type: "action",
@@ -4242,7 +4395,7 @@ const assistFieldActions = {
       title: pluginTitleShort,
       selected: isSelected
     }), [assistSupported, manageInstructions, isSelected]);
-    if (instructionsLength === 0) {
+    if (instructionsLength === 0 && !imageCaptionAction) {
       return emptyAction;
     }
     return group;
@@ -4302,13 +4455,13 @@ function useInstructionToaster(documentId, documentSchemaType) {
         return !prevTask && task.ended || !(prevTask == null ? void 0 : prevTask.ended) && task.ended;
       }).filter(task => task.ended && isAfter(addSeconds(new Date(task.ended), 30), /* @__PURE__ */new Date()));
       endedTasks == null ? void 0 : endedTasks.forEach(task => {
-        var _a2;
-        const title = getInstructionTitle(task.instruction);
+        var _a2, _b;
+        const title = (_a2 = task.title) != null ? _a2 : getInstructionTitle(task.instruction);
         if (task.reason === "error") {
           toast.push({
             title: "Failed: ".concat(title),
             status: "error",
-            description: "Instruction failed. ".concat((_a2 = task.message) != null ? _a2 : ""),
+            description: "Instruction failed. ".concat((_b = task.message) != null ? _b : ""),
             closable: true,
             duration: 1e4
           });
@@ -4355,11 +4508,11 @@ function AssistDocumentInputWrapper(props) {
     documentId
   });
 }
-function AssistDocumentInput(_ref15) {
+function AssistDocumentInput(_ref16) {
   let {
     documentId,
     ...props
-  } = _ref15;
+  } = _ref16;
   useInstructionToaster(documentId, props.schemaType);
   return /* @__PURE__ */jsx(FirstAssistedPathProvider, {
     members: props.members,
@@ -4448,12 +4601,12 @@ const assist = definePlugin(config => {
       unstable_fieldActions: prev => {
         return [...prev, assistFieldActions];
       },
-      unstable_languageFilter: (prev, _ref16) => {
+      unstable_languageFilter: (prev, _ref17) => {
         let {
           documentId,
           schema,
           schemaType
-        } = _ref16;
+        } = _ref17;
         const docSchema = schema.get(schemaType);
         return [...prev, createAssistDocumentPresence(documentId, docSchema)];
       }
@@ -4484,6 +4637,23 @@ const assist = definePlugin(config => {
           input: SafeValueInput
         }
       }
+    })(), definePlugin({
+      name: "".concat(packageName, "/generate-caption"),
+      form: {
+        components: {
+          input: props => {
+            const {
+              schemaType
+            } = props;
+            if (isImage(schemaType)) {
+              return /* @__PURE__ */jsx(ImageContextProvider, {
+                ...props
+              });
+            }
+            return props.renderDefault(props);
+          }
+        }
+      }
     })()]
   };
 });