@sanity/assist 1.0.11 → 1.1.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sanity/assist",
-  "version": "1.0.11",
+  "version": "1.1.0",
   "description": "",
   "keywords": [
     "sanity",
@@ -81,7 +81,7 @@
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
     "rimraf": "^4.4.0",
-    "sanity": "^3.13.0",
+    "sanity": "^3.14.5",
     "semantic-release": "^21.0.5",
     "styled-components": "^5.3.9",
     "typescript": "^5.1.3",
@@ -89,7 +89,7 @@
   },
   "peerDependencies": {
     "react": "^18",
-    "sanity": "^3.13.0",
+    "sanity": "^3.14.5",
     "styled-components": "^5.2"
   },
   "engines": {

package/src/assistDocument/hooks/useInstructionToaster.tsx

@@ -36,7 +36,7 @@ export function useInstructionToaster(documentId: string, documentSchemaType: Ob
         .filter((task) => task.ended && isAfter(addSeconds(new Date(task.ended), 30), new Date()))
 
       endedTasks?.forEach((task) => {
-        const title = getInstructionTitle(task.instruction)
+        const title = task.title ?? getInstructionTitle(task.instruction)
         if (task.reason === 'error') {
           toast.push({
             title: `Failed: ${title}`,

package/src/assistInspector/AssistInspector.tsx

@@ -1,6 +1,6 @@
 import {ArrowRightIcon, CloseIcon, PlayIcon, RetryIcon} from '@sanity/icons'
 import {Box, Button, Card, Flex, Spinner, Stack, Text} from '@sanity/ui'
-import {useCallback, useMemo, useState} from 'react'
+import {useCallback, useMemo, useRef} from 'react'
 import {
   DocumentInspectorProps,
   PresenceOverlay,
@@ -187,7 +187,7 @@ export function AssistInspectorWrapper(props: DocumentInspectorProps) {
 export function AssistInspector(props: DocumentInspectorProps) {
   const {params} = useAiPaneRouter()
 
-  const [boundary, setBoundary] = useState<HTMLDivElement | null>(null)
+  const boundary = useRef<HTMLDivElement | null>(null)
   const pathKey = params?.[fieldPathParam]
   const instructionKey = params?.[instructionParam]
   const documentPane = useDocumentPane()
@@ -275,7 +275,7 @@ export function AssistInspector(props: DocumentInspectorProps) {
 
   return (
     <Flex
-      ref={setBoundary}
+      ref={boundary}
       direction="column"
       height="fill"
       overflow="hidden"
@@ -290,7 +290,10 @@ export function AssistInspector(props: DocumentInspectorProps) {
             <PresenceOverlay>
               <Box padding={4}>
                 {selectedField && (
-                  <VirtualizerScrollInstanceProvider scrollElement={boundary}>
+                  <VirtualizerScrollInstanceProvider
+                    scrollElement={boundary.current}
+                    containerElement={boundary}
+                  >
                     <DocumentPaneProvider
                       paneKey={documentPane.paneKey}
                       index={documentPane.index}

package/src/assistInspector/InstructionTaskHistoryButton.tsx

@@ -111,7 +111,7 @@ export function InstructionTaskHistoryButton(props: InstructionTaskHistoryButton
           const instruction = instructions?.find((i) => i._key === task.instructionKey)
           return {
             ...task,
-            title: showTitles ? getInstructionTitle(instruction) : undefined,
+            title: showTitles ? task.title ?? getInstructionTitle(instruction) : undefined,
             cancel: () => cancelRun(task._key),
           }
         }) ?? []

package/src/components/FadeInContent.tsx

@@ -1,4 +1,4 @@
-import {ReactElement, ReactNode} from 'react'
+import {forwardRef, ReactElement, ReactNode} from 'react'
 import styled, {keyframes} from 'styled-components'
 
 const fadeIn = keyframes`
@@ -21,13 +21,20 @@ const FadeInDiv = styled.div`
   animation-timing-function: ease-in-out;
 `
 
-export function FadeInContent({
-  children,
-  durationMs = 250,
-}: {
-  children?: ReactNode
-  ms?: number
-  durationMs?: number
-}): ReactElement {
-  return <FadeInDiv style={{animationDuration: `${durationMs}ms`}}>{children}</FadeInDiv>
-}
+export const FadeInContent = forwardRef(function FadeInContent(
+  {
+    children,
+    durationMs = 250,
+  }: {
+    children?: ReactNode
+    ms?: number
+    durationMs?: number
+  },
+  ref: any
+): ReactElement {
+  return (
+    <FadeInDiv ref={ref} style={{animationDuration: `${durationMs}ms`}}>
+      {children}
+    </FadeInDiv>
+  )
+})

package/src/components/ImageContext.tsx

@@ -0,0 +1,39 @@
+import {createContext, useEffect, useMemo, useState} from 'react'
+import {InputProps, pathToString} from 'sanity'
+import {getCaptionFieldOption} from '../helpers/typeUtils'
+import {useAssistDocumentContext} from '../assistDocument/AssistDocumentContext'
+import {useApiClient, useGenerateCaption} from '../useApiClient'
+import {useAiAssistanceConfig} from '../assistLayout/AiAssistanceConfigContext'
+
+export interface ImageContextValue {
+  captionPath: string
+  assetRef?: string
+}
+
+export const ImageContext = createContext<ImageContextValue | undefined>(undefined)
+
+export function ImageContextProvider(props: InputProps) {
+  const {schemaType, path, value} = props
+  const assetRef = (value as any)?.asset?._ref
+  const [assetRefState, setAssetRefState] = useState<string | undefined>(assetRef)
+
+  const {documentId} = useAssistDocumentContext()
+  const {config} = useAiAssistanceConfig()
+  const apiClient = useApiClient(config?.__customApiClient)
+  const {generateCaption} = useGenerateCaption(apiClient)
+
+  useEffect(() => {
+    const captionField = getCaptionFieldOption(schemaType)
+    if (assetRef && documentId && captionField && assetRef !== assetRefState) {
+      setAssetRefState(assetRef)
+      generateCaption({path: pathToString([...path, captionField]), documentId: documentId})
+    }
+  }, [schemaType, path, assetRef, assetRefState, documentId, generateCaption])
+
+  const context: ImageContextValue | undefined = useMemo(() => {
+    const captionField = getCaptionFieldOption(schemaType)
+    return captionField ? {captionPath: pathToString([...path, captionField]), assetRef} : undefined
+  }, [schemaType, path, assetRef])
+
+  return <ImageContext.Provider value={context}>{props.renderDefault(props)}</ImageContext.Provider>
+}

package/src/fieldActions/assistFieldActions.tsx

@@ -20,6 +20,7 @@ import {
   useRequestRunInstruction,
 } from '../assistDocument/RequestRunInstructionProvider'
 import {PrivateIcon} from './PrivateIcon'
+import {generateCaptionsActions} from './generateCaptionActions'
 
 function node(node: DocumentFieldActionItem | DocumentFieldActionGroup) {
   return node
@@ -73,6 +74,8 @@ export const assistFieldActions: DocumentFieldAction = {
     const isPathSelected = pathKey === selectedPath
     const isSelected = isInspectorOpen && isPathSelected
 
+    const imageCaptionAction = generateCaptionsActions.useAction(props)
+
     const manageInstructions = useCallback(
       () =>
         isSelected
@@ -121,16 +124,19 @@ export const assistFieldActions: DocumentFieldAction = {
             type: 'group',
             icon: () => null,
             title: 'Run instructions',
-            children: instructions.map((instruction) =>
-              instructionItem({
-                instruction,
-                isPrivate: Boolean(instruction.userId && instruction.userId === currentUser?.id),
-                onInstructionAction,
-                hidden: isHidden,
-                documentIsNew: !!documentIsNew,
-                assistSupported,
-              })
-            ),
+            children: [
+              ...instructions.map((instruction) =>
+                instructionItem({
+                  instruction,
+                  isPrivate: Boolean(instruction.userId && instruction.userId === currentUser?.id),
+                  onInstructionAction,
+                  hidden: isHidden,
+                  documentIsNew: !!documentIsNew,
+                  assistSupported,
+                })
+              ),
+              imageCaptionAction,
+            ].filter(Boolean),
             expanded: true,
           })
         : undefined
@@ -141,6 +147,7 @@ export const assistFieldActions: DocumentFieldAction = {
       isHidden,
       documentIsNew,
       assistSupported,
+      imageCaptionAction,
     ])
 
     const instructionsLength = instructions?.length || 0
@@ -203,7 +210,7 @@ export const assistFieldActions: DocumentFieldAction = {
     )
 
     // If there are no instructions, we don't want to render the group
-    if (instructionsLength === 0) {
+    if (instructionsLength === 0 && !imageCaptionAction) {
       return emptyAction
     }
 

package/src/fieldActions/generateCaptionActions.tsx

@@ -0,0 +1,58 @@
+import {DocumentFieldAction, DocumentFieldActionGroup, DocumentFieldActionItem} from 'sanity'
+import {ImageIcon} from '@sanity/icons'
+import {useContext, useMemo} from 'react'
+import {usePathKey} from '../helpers/misc'
+import {useApiClient, useGenerateCaption} from '../useApiClient'
+import {useAiAssistanceConfig} from '../assistLayout/AiAssistanceConfigContext'
+import {useAssistDocumentContext} from '../assistDocument/AssistDocumentContext'
+import {ImageContext} from '../components/ImageContext'
+import {Box, Spinner} from '@sanity/ui'
+
+function node(node: DocumentFieldActionItem | DocumentFieldActionGroup) {
+  return node
+}
+
+export const generateCaptionsActions: DocumentFieldAction = {
+  name: 'sanity-assist-generate-captions',
+  useAction(props) {
+    const pathKey = usePathKey(props.path)
+
+    const {config} = useAiAssistanceConfig()
+    const apiClient = useApiClient(config?.__customApiClient)
+    const {generateCaption, loading} = useGenerateCaption(apiClient)
+
+    const imageContext = useContext(ImageContext)
+
+    if (imageContext && pathKey === imageContext?.captionPath) {
+      //if this is true, it is stable, and not breaking rules of hooks
+      // eslint-disable-next-line react-hooks/rules-of-hooks
+      const {documentId} = useAssistDocumentContext()
+      // eslint-disable-next-line react-hooks/rules-of-hooks
+      return useMemo(() => {
+        return node({
+          type: 'action',
+          icon: loading
+            ? () => (
+                <Box style={{height: 17}}>
+                  <Spinner style={{transform: 'translateY(6px)'}} />
+                </Box>
+              )
+            : ImageIcon,
+          title: 'Generate caption',
+          onAction: () => {
+            if (loading) {
+              return
+            }
+            generateCaption({path: pathKey, documentId: documentId ?? ''})
+          },
+          renderAsButton: true,
+          disabled: loading,
+          hidden: !imageContext.assetRef,
+        })
+      }, [generateCaption, pathKey, documentId, loading, imageContext])
+    }
+
+    // works but not supported by types
+    return undefined as unknown as DocumentFieldActionItem
+  },
+}

package/src/helpers/typeUtils.ts

@@ -1,4 +1,4 @@
-import {ArraySchemaType, SchemaType} from 'sanity'
+import {ArraySchemaType, ImageOptions, SchemaType} from 'sanity'
 
 export function isPortableTextArray(type: ArraySchemaType) {
   return type.of.find((t) => isType(t, 'block'))
@@ -13,3 +13,18 @@ export function isType(schemaType: SchemaType, typeName: string): boolean {
   }
   return isType(schemaType.type, typeName)
 }
+
+export function isImage(schemaType: SchemaType) {
+  return isType(schemaType, 'image')
+}
+
+export function getCaptionFieldOption(schemaType: SchemaType | undefined): string | undefined {
+  if (!schemaType) {
+    return undefined
+  }
+  const captionField = (schemaType.options as ImageOptions)?.captionField
+  if (captionField) {
+    return captionField
+  }
+  return getCaptionFieldOption(schemaType.type)
+}

package/src/plugin.tsx

@@ -13,6 +13,8 @@ import {packageName} from './constants'
 import {AssistDocumentInputWrapper} from './assistDocument/AssistDocumentInput'
 import {createAssistDocumentPresence} from './presence/AssistDocumentPresence'
 import {isSchemaAssistEnabled} from './helpers/assistSupported'
+import {isImage} from './helpers/typeUtils'
+import {ImageContextProvider} from './components/ImageContext'
 
 export interface AssistPluginConfig {
   /**
@@ -75,6 +77,22 @@ export const assist = definePlugin<AssistPluginConfig | void>((config) => {
         name: `${packageName}/safe-value-input`,
         form: {components: {input: SafeValueInput}},
       })(),
+
+      definePlugin({
+        name: `${packageName}/generate-caption`,
+        form: {
+          components: {
+            input: (props) => {
+              const {schemaType} = props
+
+              if (isImage(schemaType)) {
+                return <ImageContextProvider {...props} />
+              }
+              return props.renderDefault(props)
+            },
+          },
+        },
+      })(),
     ],
   }
 })

package/src/schemas/typeDefExtensions.ts

@@ -18,6 +18,7 @@ declare module 'sanity' {
   interface GeopointOptions extends AssistOptions {}
   interface ImageOptions extends AssistOptions {
     imagePromptField?: string
+    captionField?: string
   }
   interface NumberOptions extends AssistOptions {}
   interface ObjectOptions extends AssistOptions {}

package/src/types.ts

@@ -125,6 +125,7 @@ export interface InstructionTask {
   _key: string
   _type: typeof instructionTaskTypeName
   instructionKey?: string
+  title?: string
   path?: string
   started?: string
   updated?: string

package/src/useApiClient.ts

@@ -35,6 +35,59 @@ export function useApiClient(customApiClient?: (defaultClient: SanityClient) =>
   )
 }
 
+export function useGenerateCaption(apiClient: SanityClient) {
+  const [loading, setLoading] = useState(false)
+  const user = useCurrentUser()
+  const schema = useSchema()
+  const types = useMemo(() => serializeSchema(schema, {leanFormat: true}), [schema])
+  const toast = useToast()
+
+  const generateCaption = useCallback(
+    ({path, documentId}: {path: string; documentId: string}) => {
+      setLoading(true)
+
+      return apiClient
+        .request({
+          method: 'POST',
+          url: `/assist/tasks/generate-caption/${apiClient.config().dataset}?projectId=${
+            apiClient.config().projectId
+          }`,
+          body: {
+            path,
+            documentId,
+            types,
+            userId: user?.id,
+          },
+        })
+        .catch((e) => {
+          toast.push({
+            status: 'error',
+            title: 'Generate caption failed',
+            description: e.message,
+          })
+          setLoading(false)
+          throw e
+        })
+        .finally(() => {
+          // adding some artificial delay here
+          // server responds with 201 then proceeds; we dont need to allow spamming the button
+          setTimeout(() => {
+            setLoading(false)
+          }, 2000)
+        })
+    },
+    [setLoading, apiClient, toast, user, types]
+  )
+
+  return useMemo(
+    () => ({
+      generateCaption,
+      loading,
+    }),
+    [generateCaption, loading]
+  )
+}
+
 export function useGetInstructStatus(apiClient: SanityClient) {
   const [loading, setLoading] = useState(true)
   const projectClient = useClient({apiVersion: '2023-06-05'})