@lota-sdk/core 0.1.5

package/src/services/workstream.types.ts

@@ -0,0 +1,118 @@
+import { z } from 'zod'
+
+import type { WorkstreamState } from '../runtime/workstream-state'
+
+export interface Citation {
+  title?: string
+  url?: string
+  snippet?: string
+  source?: string
+  sourceId?: string
+  retrievedAt?: string
+}
+
+const WorkstreamModeSchema = z.enum(['direct', 'group'])
+export const WorkstreamStatusSchema = z.enum(['regular', 'archived'])
+const CoreWorkstreamTypeSchema = z.string()
+
+export interface NormalizedWorkstream {
+  id: string
+  title: string
+  status: 'regular' | 'archived'
+  mode: 'direct' | 'group'
+  core: boolean
+  coreType?: string
+  isRunning: boolean
+  isCompacting: boolean
+  agentId?: string | null
+  memoryBlock: string
+  createdAt: string
+  updatedAt: string
+  userId: string
+  organizationId: string
+}
+
+export interface PublicWorkstreamStateFocus {
+  kind: 'plan' | 'task' | 'question' | 'decision' | 'agent-note' | 'chat-summary'
+  text: string
+}
+
+export interface PublicWorkstreamTaskProgress {
+  total: number
+  open: number
+  inProgress: number
+  done: number
+  blocked: number
+}
+
+export interface PublicWorkstreamConstraintProgress {
+  total: number
+  approved: number
+  candidate: number
+}
+
+export interface PublicWorkstreamStateProgress {
+  hasState: boolean
+  lastUpdated: string | null
+  completionRatio: number | null
+  tasks: PublicWorkstreamTaskProgress
+  constraints: PublicWorkstreamConstraintProgress
+  keyDecisions: number
+  openQuestions: number
+  risks: number
+  artifacts: number
+  agentContributions: number
+}
+
+export interface PublicWorkstreamApprovalState {
+  approvedBy: string | null
+  approvedAt: string | null
+  approvalMessageId: string | null
+  approvalNote: string | null
+}
+
+export interface PublicWorkstreamStatePayload {
+  focus: PublicWorkstreamStateFocus | null
+  chatSummary: string | null
+  approval: PublicWorkstreamApprovalState | null
+  progress: PublicWorkstreamStateProgress
+  snapshot: WorkstreamState | null
+}
+
+export interface PublicWorkstreamDetail {
+  id: string
+  title: string
+  status: 'regular' | 'archived'
+  mode: 'direct' | 'group'
+  core: boolean
+  coreType?: string
+  isRunning: boolean
+  isCompacting: boolean
+  agentId?: string | null
+  createdAt: string
+  updatedAt: string
+  workstreamState: PublicWorkstreamStatePayload
+}
+
+export const WorkstreamSchema = z.object({
+  id: z.any(), // RecordId
+  mode: WorkstreamModeSchema.optional().default('group'),
+  core: z.boolean().optional().default(false),
+  coreType: CoreWorkstreamTypeSchema.nullish(),
+  agentId: z.string().nullish(),
+  title: z.string().nullish(),
+  status: WorkstreamStatusSchema.nullish(),
+  memoryBlock: z.string().nullish(),
+  memoryBlockSummary: z.string().nullish(),
+  activeRunId: z.string().nullish(),
+  chatSummary: z.string().nullish(),
+  lastCompactedMessageId: z.string().nullish(),
+  isCompacting: z.boolean().optional(),
+  state: z.unknown().optional(),
+  createdAt: z.coerce.date(),
+  updatedAt: z.coerce.date(),
+  userId: z.any(), // RecordId
+  organizationId: z.any(), // RecordId
+})
+
+export type WorkstreamRecord = z.infer<typeof WorkstreamSchema>

package/src/storage/attachment-parser.ts

@@ -0,0 +1,101 @@
+import { SUPPORTED_ATTACHMENT_MIME_TYPES, getAttachmentExtension } from '@lota-sdk/shared/constants/attachments'
+import mammoth from 'mammoth'
+import { PDFParse } from 'pdf-parse'
+
+const READ_FILE_PARTS_CHARS_PER_PAGE = 3500
+
+export function normalizeExtractedText(value: string): string {
+  return value.replaceAll('\u0000', '').replace(/\r\n/g, '\n').trim()
+}
+
+export function isPdfAttachmentFile(file: File): boolean {
+  const lowerName = file.name.toLowerCase()
+  const lowerType = file.type.toLowerCase()
+  return lowerType === 'application/pdf' || lowerName.endsWith('.pdf')
+}
+
+export function isDocxAttachmentFile(file: File): boolean {
+  const lowerName = file.name.toLowerCase()
+  const lowerType = file.type.toLowerCase()
+  return (
+    lowerType === 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' ||
+    lowerName.endsWith('.docx')
+  )
+}
+
+export function isTextAttachmentFile(file: File): boolean {
+  if (SUPPORTED_ATTACHMENT_MIME_TYPES.has(file.type.toLowerCase())) return true
+  const ext = getAttachmentExtension(file.name)
+  return ext === 'txt' || ext === 'md' || ext === 'markdown'
+}
+
+export async function extractPdfPages(file: File): Promise<string[]> {
+  const parser = new PDFParse({ data: new Uint8Array(await file.arrayBuffer()) })
+  try {
+    const info = await parser.getInfo()
+    const textResult = await parser.getText()
+
+    const pageTextByNumber = new Map<number, string>()
+    for (const page of textResult.pages) {
+      pageTextByNumber.set(page.num, normalizeExtractedText(page.text))
+    }
+
+    const pages: string[] = []
+    for (let pageNumber = 1; pageNumber <= info.total; pageNumber += 1) {
+      pages.push(pageTextByNumber.get(pageNumber) ?? '')
+    }
+    return pages
+  } finally {
+    await parser.destroy()
+  }
+}
+
+export async function extractDocxText(file: File): Promise<string> {
+  const buffer = Buffer.from(await file.arrayBuffer())
+  const result = await mammoth.extractRawText({ buffer })
+  return normalizeExtractedText(result.value)
+}
+
+export async function extractAttachmentText(file: File): Promise<string> {
+  if (isTextAttachmentFile(file)) {
+    return normalizeExtractedText(await file.text())
+  }
+  if (isPdfAttachmentFile(file)) {
+    return normalizeExtractedText((await extractPdfPages(file)).join('\n\n'))
+  }
+  if (isDocxAttachmentFile(file)) {
+    return await extractDocxText(file)
+  }
+  return ''
+}
+
+export function splitExtractedTextIntoPages(extractedText: string): string[] {
+  const normalized = extractedText.replace(/\r\n/g, '\n').trim()
+  if (!normalized) return []
+
+  const explicitPages = normalized
+    .split(/\f+/)
+    .map((value) => value.trim())
+    .filter((value) => value.length > 0)
+  if (explicitPages.length > 1) return explicitPages
+
+  const pages: string[] = []
+  let cursor = 0
+  while (cursor < normalized.length) {
+    let end = Math.min(cursor + READ_FILE_PARTS_CHARS_PER_PAGE, normalized.length)
+    if (end < normalized.length) {
+      const breakAt = normalized.lastIndexOf('\n', end)
+      if (breakAt > cursor + Math.floor(READ_FILE_PARTS_CHARS_PER_PAGE * 0.6)) {
+        end = breakAt
+      }
+    }
+
+    const pageText = normalized.slice(cursor, end).trim()
+    if (pageText) pages.push(pageText)
+
+    cursor = end
+    while (cursor < normalized.length && normalized[cursor] === '\n') cursor += 1
+  }
+
+  return pages
+}

package/src/storage/attachment-storage.service.ts

@@ -0,0 +1,391 @@
+import { inferContentType } from '@lota-sdk/shared/constants/attachments'
+import { S3Client } from 'bun'
+
+import { env } from '../config/env-shapes'
+import { serverLogger } from '../config/logger'
+import { readString } from '../utils/string'
+import {
+  extractAttachmentText,
+  extractPdfPages,
+  isPdfAttachmentFile,
+  splitExtractedTextIntoPages,
+} from './attachment-parser'
+import type {
+  MessagePartLike,
+  ReadableUploadMetadata,
+  ReadableUploadPageMode,
+  ReadableUploadPagePart,
+} from './attachments.types'
+import {
+  buildOrganizationDocumentStorageKey,
+  buildUploadStorageKey,
+  buildUploadStoragePrefix,
+  readNonNegativeInteger,
+  readRecord,
+} from './attachments.utils'
+
+const READ_FILE_PARTS_PAGES_PER_PART = 25
+
+export type UploadedWorkstreamAttachment = {
+  filename: string
+  mediaType: string
+  sizeBytes: number
+  storageKey: string
+  url: string
+  expiresInSeconds: number
+}
+
+type AttachmentContextCandidate = { storageKey: string; name: string; contentType: string; sizeBytes: number | null }
+
+export class AttachmentStorageService {
+  private readonly client: S3Client
+
+  constructor() {
+    this.client = new S3Client({
+      accessKeyId: env.S3_ACCESS_KEY_ID,
+      secretAccessKey: env.S3_SECRET_ACCESS_KEY,
+      bucket: env.S3_BUCKET,
+      endpoint: env.S3_ENDPOINT,
+      region: env.S3_REGION,
+    })
+  }
+
+  getAttachmentUrl(storageKey: string): string {
+    return this.client.file(storageKey).presign({ expiresIn: env.ATTACHMENT_URL_EXPIRES_IN })
+  }
+
+  async writeOrganizationDocument({
+    orgId,
+    namespace,
+    relativePath,
+    content,
+    contentType,
+  }: {
+    orgId: string
+    namespace: string
+    relativePath: string
+    content: string
+    contentType: string
+  }): Promise<{ storageKey: string; sizeBytes: number }> {
+    const storageKey = buildOrganizationDocumentStorageKey({ orgId, namespace, relativePath })
+    const sizeBytes = Buffer.byteLength(content, 'utf8')
+
+    await this.client.file(storageKey).write(new Blob([content]), { type: contentType })
+
+    return { storageKey, sizeBytes }
+  }
+
+  async uploadOrganizationDocument({
+    file,
+    orgId,
+    namespace,
+    relativePath,
+  }: {
+    file: File
+    orgId: string
+    namespace: string
+    relativePath: string
+  }): Promise<UploadedWorkstreamAttachment> {
+    const filename = file.name || 'document'
+    const mediaType = file.type || inferContentType(filename)
+    const storageKey = buildOrganizationDocumentStorageKey({
+      orgId,
+      namespace,
+      relativePath: relativePath.trim().length > 0 ? relativePath : filename,
+    })
+
+    await this.client.file(storageKey).write(file, { type: mediaType })
+
+    return {
+      filename,
+      mediaType,
+      sizeBytes: file.size,
+      storageKey,
+      url: this.getAttachmentUrl(storageKey),
+      expiresInSeconds: env.ATTACHMENT_URL_EXPIRES_IN,
+    }
+  }
+
+  async uploadWorkstreamAttachment({
+    file,
+    orgId,
+    userId,
+  }: {
+    file: File
+    orgId: string
+    userId: string
+  }): Promise<UploadedWorkstreamAttachment> {
+    const filename = file.name || 'attachment'
+    const mediaType = file.type || inferContentType(filename)
+    const sizeBytes = file.size
+    const storageKey = buildUploadStorageKey({ orgId, userId, filename, uploadId: Bun.randomUUIDv7() })
+
+    await this.client.file(storageKey).write(file, { type: mediaType })
+
+    return {
+      filename,
+      mediaType,
+      sizeBytes,
+      storageKey,
+      url: this.getAttachmentUrl(storageKey),
+      expiresInSeconds: env.ATTACHMENT_URL_EXPIRES_IN,
+    }
+  }
+
+  hydrateSignedFileUrlsInMessageParts({
+    parts,
+    orgId,
+    userId,
+  }: {
+    parts: readonly MessagePartLike[]
+    orgId: string
+    userId: string
+  }): MessagePartLike[] {
+    const storagePrefix = buildUploadStoragePrefix({ orgId, userId })
+
+    return parts.map((part) => {
+      if (part.type !== 'file') return part
+
+      const providerMetadata = readRecord(part.providerMetadata)
+      const lotaMetadata = readRecord(providerMetadata?.lota)
+      const storageKey = lotaMetadata?.attachmentStorageKey
+      if (typeof storageKey !== 'string' || !storageKey.startsWith(storagePrefix)) return part
+
+      return { ...part, url: this.getAttachmentUrl(storageKey) }
+    })
+  }
+
+  listReadableUploadsFromMessages({
+    messages,
+    orgId,
+    userId,
+  }: {
+    messages: ReadonlyArray<{ parts: readonly MessagePartLike[] }>
+    orgId: string
+    userId: string
+  }): ReadableUploadMetadata[] {
+    const uploadsByStorageKey = new Map<
+      string,
+      { storageKey: string; filename: string; mediaType: string; sizeBytes: number | null }
+    >()
+
+    for (const message of messages) {
+      const candidates = this.collectAttachmentContextCandidates({ parts: message.parts, orgId, userId })
+      for (const candidate of candidates) {
+        if (uploadsByStorageKey.has(candidate.storageKey)) continue
+        uploadsByStorageKey.set(candidate.storageKey, {
+          storageKey: candidate.storageKey,
+          filename: candidate.name,
+          mediaType: candidate.contentType,
+          sizeBytes: candidate.sizeBytes,
+        })
+      }
+    }
+
+    return [...uploadsByStorageKey.values()].map((upload) => ({
+      storageKey: upload.storageKey,
+      filename: upload.filename,
+      mediaType: upload.mediaType,
+      sizeBytes: upload.sizeBytes,
+    }))
+  }
+
+  async extractStoredAttachmentText({
+    storageKey,
+    name,
+    contentType,
+  }: {
+    storageKey: string
+    name: string
+    contentType: string
+  }): Promise<string> {
+    try {
+      const s3File = this.client.file(storageKey)
+      const fileLike = new File([await s3File.arrayBuffer()], name, { type: contentType })
+      return await extractAttachmentText(fileLike)
+    } catch (error) {
+      serverLogger.warn`Failed to extract stored attachment text: ${error}`
+      return ''
+    }
+  }
+
+  async extractStoredAttachmentPages({
+    storageKey,
+    name,
+    contentType,
+  }: {
+    storageKey: string
+    name: string
+    contentType: string
+  }): Promise<{ pageMode: ReadableUploadPageMode; pages: string[] }> {
+    try {
+      const s3File = this.client.file(storageKey)
+      const fileLike = new File([await s3File.arrayBuffer()], name, { type: contentType })
+
+      if (isPdfAttachmentFile(fileLike)) {
+        const pages = await extractPdfPages(fileLike)
+        return { pageMode: 'pdf', pages }
+      }
+
+      const extractedText = await extractAttachmentText(fileLike)
+      return { pageMode: 'logical', pages: splitExtractedTextIntoPages(extractedText) }
+    } catch (error) {
+      serverLogger.warn`Failed to extract attachment pages from storage: ${error}`
+      return { pageMode: 'logical', pages: [] }
+    }
+  }
+
+  async readFilePartsFromUpload({
+    upload,
+    orgId,
+    userId,
+    part = 1,
+    pagesPerPart = READ_FILE_PARTS_PAGES_PER_PART,
+  }: {
+    upload: ReadableUploadMetadata
+    orgId: string
+    userId: string
+    part?: number
+    pagesPerPart?: number
+  }): Promise<{
+    pageMode: ReadableUploadPageMode
+    totalPages: number
+    fullPageLength: number
+    currentPart: number
+    totalParts: number
+    hasNextPart: boolean
+    hasPreviousPart: boolean
+    pagesPerPart: number
+    pageStart: number
+    pageEnd: number
+    data: ReadableUploadPagePart[]
+  }> {
+    this.assertUploadOwnership({ storageKey: upload.storageKey, orgId, userId })
+
+    const parsed = await this.extractStoredAttachmentPages({
+      storageKey: upload.storageKey,
+      name: upload.filename,
+      contentType: upload.mediaType,
+    })
+    const totalPages = parsed.pages.length
+    if (totalPages === 0) {
+      return {
+        pageMode: parsed.pageMode,
+        totalPages: 0,
+        fullPageLength: 0,
+        currentPart: 1,
+        totalParts: 0,
+        hasNextPart: false,
+        hasPreviousPart: false,
+        pagesPerPart,
+        pageStart: 1,
+        pageEnd: 0,
+        data: [],
+      }
+    }
+
+    if (!Number.isInteger(pagesPerPart) || pagesPerPart <= 0) {
+      throw new Error('pagesPerPart must be a positive integer.')
+    }
+    if (!Number.isInteger(part) || part <= 0) {
+      throw new Error('part must be an integer greater than or equal to 1.')
+    }
+
+    const totalParts = Math.ceil(totalPages / pagesPerPart)
+    if (part > totalParts) {
+      throw new Error(`part ${part} exceeds total parts ${totalParts}.`)
+    }
+
+    const pageStart = (part - 1) * pagesPerPart + 1
+    const pageEnd = Math.min(pageStart + pagesPerPart - 1, totalPages)
+
+    const data = parsed.pages
+      .slice(pageStart - 1, pageEnd)
+      .map((text, index) => ({ pageNumber: pageStart + index, text, charCount: text.length }))
+
+    return {
+      pageMode: parsed.pageMode,
+      totalPages,
+      fullPageLength: totalPages,
+      currentPart: part,
+      totalParts,
+      hasNextPart: part < totalParts,
+      hasPreviousPart: part > 1,
+      pagesPerPart,
+      pageStart,
+      pageEnd,
+      data,
+    }
+  }
+
+  private assertUploadOwnership({
+    storageKey,
+    orgId,
+    userId,
+  }: {
+    storageKey: string
+    orgId: string
+    userId: string
+  }): void {
+    const storagePrefix = buildUploadStoragePrefix({ orgId, userId })
+    if (!storageKey.startsWith(storagePrefix)) {
+      throw new Error('Upload does not belong to the current organization/user scope.')
+    }
+  }
+
+  private collectAttachmentContextCandidates({
+    parts,
+    orgId,
+    userId,
+  }: {
+    parts: readonly MessagePartLike[]
+    orgId: string
+    userId: string
+  }): AttachmentContextCandidate[] {
+    const storagePrefix = buildUploadStoragePrefix({ orgId, userId })
+    const candidates: AttachmentContextCandidate[] = []
+
+    for (const part of parts) {
+      if (part.type !== 'file') continue
+
+      const providerMetadata = readRecord(part.providerMetadata)
+      const lotaMetadata = readRecord(providerMetadata?.lota)
+      const storageKey = readString(lotaMetadata?.attachmentStorageKey)
+      if (!storageKey || !storageKey.startsWith(storagePrefix)) continue
+
+      const name = readString(part.filename) ?? this.readFilenameFromStorageKey(storageKey) ?? 'Attachment'
+      const contentType = readString(part.mediaType) ?? inferContentType(name)
+      const sizeBytes = readNonNegativeInteger(lotaMetadata?.attachmentSizeBytes)
+      candidates.push({ storageKey, name, contentType, sizeBytes })
+    }
+
+    return candidates
+  }
+
+  private readFilenameFromStorageKey(storageKey: string): string | null {
+    const slashIndex = storageKey.lastIndexOf('/')
+    if (slashIndex < 0) return null
+    const rawName = storageKey.slice(slashIndex + 1).trim()
+    if (!rawName) return null
+
+    const dashIndex = rawName.indexOf('-')
+    if (dashIndex <= 0 || dashIndex === rawName.length - 1) return rawName
+    return rawName.slice(dashIndex + 1)
+  }
+}
+
+let _attachmentStorageService: AttachmentStorageService | undefined
+
+/** @lintignore */
+export function getAttachmentStorageService(): AttachmentStorageService {
+  if (!_attachmentStorageService) {
+    _attachmentStorageService = new AttachmentStorageService()
+  }
+  return _attachmentStorageService
+}
+
+export const attachmentStorageService = new Proxy({} as AttachmentStorageService, {
+  get(_target, prop: string) {
+    return (getAttachmentStorageService() as unknown as Record<string, unknown>)[prop]
+  },
+})

package/src/storage/attachments.types.ts

@@ -0,0 +1,11 @@
+export type ReadableUploadMetadata = {
+  storageKey: string
+  filename: string
+  mediaType: string
+  sizeBytes: number | null
+}
+
+export type ReadableUploadPagePart = { pageNumber: number; text: string; charCount: number }
+export type ReadableUploadPageMode = 'pdf' | 'logical'
+
+export type { MessagePartLike } from '../runtime/chat-types'

package/src/storage/attachments.utils.ts

@@ -0,0 +1,58 @@
+function sanitizeFilename(name: string): string {
+  const ascii = name.replace(/[^\x20-\x7E]/g, '')
+  return ascii.replace(/[\\/:"*?<>|]+/g, '-').trim()
+}
+
+function toSafeSegment(value: string): string {
+  const cleaned = value.replace(/[^a-zA-Z0-9._-]/g, '-')
+  return cleaned.length > 0 ? cleaned : 'unknown'
+}
+
+function sanitizeRelativePath(path: string): string {
+  return path
+    .split('/')
+    .map((segment) => sanitizeFilename(segment))
+    .filter((segment) => segment.length > 0)
+    .join('/')
+}
+
+export function buildUploadStoragePrefix(params: { orgId: string; userId: string }): string {
+  const safeOrgId = toSafeSegment(params.orgId)
+  const safeUserId = toSafeSegment(params.userId)
+  return `${safeOrgId}/${safeUserId}/uploads/`
+}
+
+export function buildUploadStorageKey(params: {
+  orgId: string
+  userId: string
+  filename: string
+  uploadId: string
+}): string {
+  const safeUploadId = toSafeSegment(params.uploadId)
+  const safeName = sanitizeFilename(params.filename) || 'attachment'
+  return `${buildUploadStoragePrefix({ orgId: params.orgId, userId: params.userId })}${safeUploadId}-${safeName}`
+}
+
+function buildOrganizationDocumentStoragePrefix(params: { orgId: string; namespace: string }): string {
+  const safeOrgId = toSafeSegment(params.orgId)
+  const safeNamespace = toSafeSegment(params.namespace)
+  return `${safeOrgId}/organization-documents/${safeNamespace}/`
+}
+
+export function buildOrganizationDocumentStorageKey(params: {
+  orgId: string
+  namespace: string
+  relativePath: string
+}): string {
+  const relativePath = sanitizeRelativePath(params.relativePath)
+  return `${buildOrganizationDocumentStoragePrefix({ orgId: params.orgId, namespace: params.namespace })}${relativePath || 'document.txt'}`
+}
+
+export function readRecord(value: unknown): Record<string, unknown> | null {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) return null
+  return value as Record<string, unknown>
+}
+
+export function readNonNegativeInteger(value: unknown): number | null {
+  return Number.isInteger(value) && typeof value === 'number' && value >= 0 ? value : null
+}