@hyperspell/openclaw-hyperspell 0.4.1 → 0.5.0

package/graph/ops.ts

@@ -0,0 +1,253 @@
+import * as fs from "node:fs"
+import * as path from "node:path"
+import type { HyperspellClient } from "../client.ts"
+import { log } from "../logger.ts"
+import { NetworkStateManager } from "./state.ts"
+
+const ENTITY_TYPES = ["people", "projects", "organizations", "topics"] as const
+export type EntityType = (typeof ENTITY_TYPES)[number]
+
+export interface SourceMemories {
+  [source: string]: string[]
+}
+
+export interface ScannedMemory {
+  resourceId: string
+  source: string
+  title: string | null
+  summary: string
+}
+
+export interface WriteEntityParams {
+  type: EntityType
+  slug: string
+  name: string
+  description: string
+  relationships?: string[]
+  sourceMemories?: SourceMemories
+  email?: string
+  phone?: string
+  domain?: string
+}
+
+export function slugify(name: string): string {
+  return name
+    .toLowerCase()
+    .trim()
+    .replace(/[^a-z0-9]+/g, "-")
+    .replace(/^-|-$/g, "")
+}
+
+function summarizeMemoryData(data: unknown[], source: string): string {
+  if (!Array.isArray(data) || data.length === 0) return ""
+
+  const items = data as Array<Record<string, unknown>>
+
+  if (source === "slack" || source === "google_mail") {
+    const senders = new Map<string, string>()
+    const messages: string[] = []
+    for (const item of items) {
+      const sender = item.sender as Record<string, unknown> | undefined
+      if (sender?.name && sender?.email) {
+        senders.set(String(sender.email), String(sender.name))
+      }
+      if (item.content && messages.length < 5) {
+        messages.push(String(item.content).slice(0, 200))
+      }
+    }
+    const parts: string[] = []
+    if (senders.size > 0) {
+      parts.push(`Participants: ${[...senders.entries()].map(([e, n]) => `${n} <${e}>`).join(", ")}`)
+    }
+    if (messages.length > 0) {
+      parts.push(`Recent messages:\n${messages.join("\n---\n")}`)
+    }
+    return parts.join("\n\n")
+  }
+
+  if (source === "notion") {
+    const parts: string[] = []
+    for (const item of items.slice(0, 10)) {
+      if (item.__type === "Title" && item.text) {
+        parts.push(`## ${item.text}`)
+      } else if (item.__type === "Markdown" && item.text) {
+        parts.push(String(item.text).slice(0, 300))
+      } else if (item.__type === "Table" && item.table_rows) {
+        const rows = item.table_rows as string[][]
+        if (rows[0]) parts.push(`Table: ${rows[0].join(" | ")}`)
+      }
+    }
+    return parts.join("\n")
+  }
+
+  const parts: string[] = []
+  for (const item of items.slice(0, 10)) {
+    if (item.text) {
+      parts.push(String(item.text).slice(0, 300))
+    }
+  }
+  return parts.join("\n")
+}
+
+export async function scanMemories(
+  client: HyperspellClient,
+  stateManager: NetworkStateManager,
+  batchSize: number,
+): Promise<ScannedMemory[]> {
+  const unprocessed: ScannedMemory[] = []
+
+  for await (const mem of client.listMemories()) {
+    if (stateManager.isProcessed(mem.resourceId)) continue
+    if (mem.metadata?.graph_entity === "true" || mem.metadata?.graph_entity === true) continue
+    if ((mem.metadata?.status as string) !== "completed") continue
+
+    let summary = ""
+    try {
+      const full = await client.getMemory(mem.resourceId, mem.source)
+      const data = full.data as unknown[] | undefined
+
+      const participants = full.participants as Array<{ name?: string; email?: string }> | undefined
+      const participantLine = participants?.length
+        ? `Participants: ${participants.map((p) => `${p.name || ""} <${p.email || ""}>`).join(", ")}`
+        : ""
+
+      const dataSummary = data ? summarizeMemoryData(data, mem.source) : ""
+      summary = [participantLine, dataSummary].filter(Boolean).join("\n\n")
+    } catch {
+      summary = "(content unavailable)"
+    }
+
+    unprocessed.push({
+      resourceId: mem.resourceId,
+      source: mem.source,
+      title: mem.title,
+      summary: summary.slice(0, 1000),
+    })
+
+    if (unprocessed.length >= batchSize) break
+  }
+
+  return unprocessed
+}
+
+export function formatScanResults(memories: ScannedMemory[], processedCount: number, lastScan: string | null): string {
+  if (memories.length === 0) {
+    return `No unprocessed memories found. ${processedCount} memories already processed. Last scan: ${lastScan || "never"}`
+  }
+
+  const formatted = memories
+    .map((m) => {
+      const lines = [`[${m.source}] ${m.title || "(untitled)"} (id: ${m.resourceId})`]
+      if (m.summary) lines.push(m.summary)
+      return lines.join("\n")
+    })
+    .join("\n\n---\n\n")
+
+  return `Found ${memories.length} unprocessed memories:\n\n${formatted}`
+}
+
+export function writeEntity(workspaceDir: string, params: WriteEntityParams): string {
+  const slug = slugify(params.slug)
+  const dir = path.join(workspaceDir, "memory", params.type)
+  const filePath = path.join(dir, `${slug}.md`)
+
+  fs.mkdirSync(dir, { recursive: true })
+
+  // Check for existing file and merge
+  let existingSourceMemories: SourceMemories = {}
+  let existingRelationships: string[] = []
+  let hyperspellId = ""
+
+  if (fs.existsSync(filePath)) {
+    const existing = fs.readFileSync(filePath, "utf-8")
+    const fmMatch = existing.match(/^---\n([\s\S]*?)\n---\n?/)
+    if (fmMatch) {
+      const fmText = fmMatch[1]
+      for (const line of fmText.split("\n")) {
+        const idx = line.indexOf(":")
+        if (idx <= 0) continue
+        const key = line.slice(0, idx).trim()
+        const val = line.slice(idx + 1).trim()
+        if (key === "hyperspell_id") hyperspellId = val
+        if (key === "source_memories") {
+          try { existingSourceMemories = JSON.parse(val) } catch {}
+        }
+        if (key === "relationships") {
+          try { existingRelationships = JSON.parse(val) } catch {}
+        }
+      }
+    }
+  }
+
+  // Merge source memories
+  const mergedSources: SourceMemories = { ...existingSourceMemories }
+  if (params.sourceMemories) {
+    for (const [source, ids] of Object.entries(params.sourceMemories)) {
+      const existing = mergedSources[source] || []
+      const merged = [...new Set([...existing, ...ids])]
+      mergedSources[source] = merged
+    }
+  }
+
+  // Merge relationships
+  const mergedRelationships = [
+    ...new Set([...existingRelationships, ...(params.relationships || [])]),
+  ]
+
+  // Build frontmatter
+  const fm: Record<string, string> = {
+    title: params.name,
+    type: params.type.slice(0, -1), // "people" → "person"
+    graph_entity: "true",
+    source_memories: JSON.stringify(mergedSources),
+    last_extracted: new Date().toISOString(),
+  }
+  if (hyperspellId) fm.hyperspell_id = hyperspellId
+  if (mergedRelationships.length > 0) {
+    fm.relationships = JSON.stringify(mergedRelationships)
+  }
+  if (params.email) fm.email = params.email
+  if (params.phone) fm.phone = params.phone
+  if (params.domain) fm.domain = params.domain
+
+  // Build body
+  const bodyParts = [`# ${params.name}\n`, params.description]
+
+  const contactParts: string[] = []
+  if (params.email) contactParts.push(`- Email: ${params.email}`)
+  if (params.phone) contactParts.push(`- Phone: ${params.phone}`)
+  if (params.domain) contactParts.push(`- Domain: ${params.domain}`)
+  if (contactParts.length > 0) {
+    bodyParts.push("\n## Contact\n")
+    bodyParts.push(...contactParts)
+  }
+
+  if (mergedRelationships.length > 0) {
+    bodyParts.push("\n## Relationships\n")
+    for (const rel of mergedRelationships) {
+      const [relationship, target] = rel.split(":")
+      if (target) {
+        const targetName = target.split("/").pop()?.replace(/-/g, " ") || target
+        bodyParts.push(`- ${relationship}: [${targetName}](../${target}.md)`)
+      } else {
+        bodyParts.push(`- ${rel}`)
+      }
+    }
+  }
+
+  // Build file content
+  const fmLines = Object.entries(fm).map(([k, v]) => `${k}: ${v}`)
+  const content = `---\n${fmLines.join("\n")}\n---\n${bodyParts.join("\n")}\n`
+
+  fs.writeFileSync(filePath, content)
+  log.info(`Wrote entity: ${params.type}/${slug}.md`)
+
+  return `${params.type}/${slug}.md`
+}
+
+export function completeMemories(stateManager: NetworkStateManager, memoryIds: string[]): { newCount: number; totalCount: number } {
+  const newCount = stateManager.markProcessed(memoryIds)
+  stateManager.updateLastScan()
+  stateManager.save()
+  return { newCount, totalCount: stateManager.getProcessedCount() }
+}

package/graph/state.ts

@@ -0,0 +1,79 @@
+import * as fs from "node:fs"
+import * as path from "node:path"
+import { log } from "../logger.ts"
+
+const STATE_VERSION = 1
+const STATE_FILENAME = ".network-state.json"
+
+export interface NetworkState {
+  processedIds: Record<string, string> // resource_id → ISO timestamp
+  lastScanAt: string | null
+  version: number
+}
+
+export class NetworkStateManager {
+  private statePath: string
+  private state: NetworkState
+
+  constructor(workspaceDir: string) {
+    const memoryDir = path.join(workspaceDir, "memory")
+    fs.mkdirSync(memoryDir, { recursive: true })
+    this.statePath = path.join(memoryDir, STATE_FILENAME)
+    this.state = this.load()
+  }
+
+  private load(): NetworkState {
+    try {
+      if (fs.existsSync(this.statePath)) {
+        const raw = fs.readFileSync(this.statePath, "utf-8")
+        const parsed = JSON.parse(raw)
+        if (parsed.version === STATE_VERSION) {
+          return parsed
+        }
+        log.warn(`Network state version mismatch (got ${parsed.version}, want ${STATE_VERSION}), resetting`)
+      }
+    } catch (err) {
+      log.warn("Failed to load network state, starting fresh", err)
+    }
+    return { processedIds: {}, lastScanAt: null, version: STATE_VERSION }
+  }
+
+  save(): void {
+    const tmpPath = this.statePath + ".tmp"
+    try {
+      fs.writeFileSync(tmpPath, JSON.stringify(this.state, null, 2))
+      fs.renameSync(tmpPath, this.statePath)
+    } catch (err) {
+      log.error("Failed to save network state", err)
+      try { fs.unlinkSync(tmpPath) } catch {}
+    }
+  }
+
+  isProcessed(resourceId: string): boolean {
+    return resourceId in this.state.processedIds
+  }
+
+  markProcessed(resourceIds: string[]): number {
+    let count = 0
+    const now = new Date().toISOString()
+    for (const id of resourceIds) {
+      if (!(id in this.state.processedIds)) {
+        this.state.processedIds[id] = now
+        count++
+      }
+    }
+    return count
+  }
+
+  updateLastScan(): void {
+    this.state.lastScanAt = new Date().toISOString()
+  }
+
+  getProcessedCount(): number {
+    return Object.keys(this.state.processedIds).length
+  }
+
+  getLastScanAt(): string | null {
+    return this.state.lastScanAt
+  }
+}

package/graph/tools.ts

@@ -0,0 +1,117 @@
+import { Type } from "@sinclair/typebox"
+import type { OpenClawPluginApi } from "openclaw/plugin-sdk"
+import type { HyperspellClient } from "../client.ts"
+import type { HyperspellConfig } from "../config.ts"
+import { getWorkspaceDir } from "../config.ts"
+import { log } from "../logger.ts"
+import { NetworkStateManager } from "./state.ts"
+import { scanMemories, formatScanResults, writeEntity, completeMemories } from "./ops.ts"
+import type { EntityType, SourceMemories } from "./ops.ts"
+
+const ENTITY_TYPES = ["people", "projects", "organizations", "topics"] as const
+
+export function registerNetworkTools(
+  api: OpenClawPluginApi,
+  client: HyperspellClient,
+  cfg: HyperspellConfig,
+): void {
+  const workspaceDir = getWorkspaceDir()
+  const stateManager = new NetworkStateManager(workspaceDir)
+
+  api.registerTool(
+    {
+      name: "hyperspell_network_scan",
+      label: "Memory Network Scan",
+      description:
+        "Scan Hyperspell memories and return a batch of unprocessed ones with their content summaries. Use this to find new memories that need entity extraction.",
+      parameters: Type.Object({
+        batchSize: Type.Optional(
+          Type.Number({ description: "Max memories to return (default: 20)" }),
+        ),
+      }),
+      async execute(_toolCallId: string, params: { batchSize?: number }) {
+        const batchSize = params.batchSize ?? cfg.knowledgeGraph.batchSize
+        try {
+          const memories = await scanMemories(client, stateManager, batchSize)
+          const text = formatScanResults(memories, stateManager.getProcessedCount(), stateManager.getLastScanAt())
+          return {
+            content: [{ type: "text" as const, text }],
+            details: { count: memories.length, memories },
+          }
+        } catch (err) {
+          log.error("network scan failed", err)
+          return {
+            content: [{ type: "text" as const, text: `Scan failed: ${err instanceof Error ? err.message : String(err)}` }],
+          }
+        }
+      },
+    },
+    { name: "hyperspell_network_scan" },
+  )
+
+  api.registerTool(
+    {
+      name: "hyperspell_network_write",
+      label: "Memory Network Write",
+      description:
+        "Write or update an entity file in the memory network. Creates markdown files in memory/people/, memory/projects/, memory/organizations/, or memory/topics/.",
+      parameters: Type.Object({
+        type: Type.Union(ENTITY_TYPES.map((t) => Type.Literal(t)), {
+          description: "Entity type: people, projects, organizations, or topics",
+        }),
+        slug: Type.String({ description: "URL-safe identifier (lowercase with hyphens, e.g. 'alice-chen')" }),
+        name: Type.String({ description: "Display name of the entity" }),
+        description: Type.String({ description: "Description of the entity" }),
+        relationships: Type.Optional(
+          Type.Array(Type.String(), {
+            description: "Relationships in format 'relationship:type/slug', e.g. 'works-at:organizations/hyperspell'",
+          }),
+        ),
+        sourceMemories: Type.Optional(
+          Type.Record(Type.String(), Type.Array(Type.String()), {
+            description: "Source memories by provider, e.g. { slack: ['C073WR69EPM'], google_mail: ['19bbe68026553623'] }",
+          }),
+        ),
+        email: Type.Optional(Type.String({ description: "Email address (for people)" })),
+        phone: Type.Optional(Type.String({ description: "Phone number (for people)" })),
+        domain: Type.Optional(Type.String({ description: "Domain/homepage (for organizations)" })),
+      }),
+      async execute(
+        _toolCallId: string,
+        params: { type: EntityType; slug: string; name: string; description: string; relationships?: string[]; sourceMemories?: SourceMemories; email?: string; phone?: string; domain?: string },
+      ) {
+        try {
+          const result = writeEntity(workspaceDir, params)
+          return { content: [{ type: "text" as const, text: `Wrote ${result} (${params.name})` }] }
+        } catch (err) {
+          log.error(`network write failed: ${params.type}/${params.slug}`, err)
+          return { content: [{ type: "text" as const, text: `Write failed: ${err instanceof Error ? err.message : String(err)}` }] }
+        }
+      },
+    },
+    { name: "hyperspell_network_write" },
+  )
+
+  api.registerTool(
+    {
+      name: "hyperspell_network_complete",
+      label: "Memory Network Complete",
+      description: "Mark a batch of memory IDs as processed so they won't appear in future scans.",
+      parameters: Type.Object({
+        memoryIds: Type.Array(Type.String(), { description: "List of resource_ids to mark as processed" }),
+      }),
+      async execute(_toolCallId: string, params: { memoryIds: string[] }) {
+        try {
+          const { newCount, totalCount } = completeMemories(stateManager, params.memoryIds)
+          return {
+            content: [{ type: "text" as const, text: `Marked ${newCount} new memories as processed (${totalCount} total). Last scan: ${stateManager.getLastScanAt()}` }],
+          }
+        } catch (err) {
+          log.error("network complete failed", err)
+          return { content: [{ type: "text" as const, text: `Complete failed: ${err instanceof Error ? err.message : String(err)}` }] }
+        }
+      },
+    },
+    { name: "hyperspell_network_complete" },
+  )
+}

package/index.ts

@@ -8,6 +8,7 @@ import { buildFileSyncHandler, syncMemoriesOnStartup } from "./hooks/memory-sync
 import { initLogger } from "./logger.ts"
 import { registerRememberTool } from "./tools/remember.ts"
 import { registerSearchTool } from "./tools/search.ts"
+import { registerNetworkTools } from "./graph/index.ts"
 
 export default {
   id: "openclaw-hyperspell",
@@ -84,6 +85,11 @@ export default {
       api.on("file_changed", fileSyncHandler)
     }
 
+    // Register memory network tools
+    if (cfg.knowledgeGraph.enabled) {
+      registerNetworkTools(api, client, cfg)
+    }
+
     // Register slash commands
     registerCommands(api, client, cfg)
 

package/openclaw.plugin.json

@@ -40,6 +40,11 @@
       "label": "Sync Memory Files",
       "help": "Automatically sync markdown files in workspace/memory/ to Hyperspell",
       "advanced": true
+    },
+    "knowledgeGraph": {
+      "label": "Memory Network",
+      "help": "Extract entities (people, projects, orgs, topics) from memories into structured markdown files. Requires a cron job for periodic scanning.",
+      "advanced": true
     }
   },
   "configSchema": {
@@ -68,6 +73,14 @@
       },
       "syncMemories": {
         "type": "boolean"
+      },
+      "knowledgeGraph": {
+        "type": "object",
+        "properties": {
+          "enabled": { "type": "boolean" },
+          "scanIntervalMinutes": { "type": "number", "minimum": 5, "maximum": 1440 },
+          "batchSize": { "type": "number", "minimum": 5, "maximum": 100 }
+        }
       }
     },
     "required": []

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hyperspell/openclaw-hyperspell",
-  "version": "0.4.1",
+  "version": "0.5.0",
   "type": "module",
   "description": "OpenClaw Hyperspell memory plugin",
   "license": "MIT",
@@ -10,6 +10,8 @@
     "commands/",
     "hooks/",
     "tools/",
+    "sync/",
+    "graph/",
     "lib/",
     "types/",
     "openclaw.plugin.json",

package/sync/markdown.ts

@@ -0,0 +1,183 @@
+import * as fs from "node:fs"
+import * as path from "node:path"
+import type { HyperspellClient } from "../client.ts"
+import { log } from "../logger.ts"
+
+const FRONTMATTER_REGEX = /^---\n([\s\S]*?)\n---\n?/
+
+interface MarkdownFile {
+  filePath: string
+  title: string
+  content: string
+  hyperspellId: string | null
+}
+
+/**
+ * Parse frontmatter from markdown content
+ */
+function parseFrontmatter(content: string): { frontmatter: Record<string, string>; body: string } {
+  const match = content.match(FRONTMATTER_REGEX)
+  if (!match) {
+    return { frontmatter: {}, body: content }
+  }
+
+  const frontmatterText = match[1]
+  const body = content.slice(match[0].length)
+  const frontmatter: Record<string, string> = {}
+
+  for (const line of frontmatterText.split("\n")) {
+    const colonIndex = line.indexOf(":")
+    if (colonIndex > 0) {
+      const key = line.slice(0, colonIndex).trim()
+      const value = line.slice(colonIndex + 1).trim()
+      frontmatter[key] = value
+    }
+  }
+
+  return { frontmatter, body }
+}
+
+/**
+ * Serialize frontmatter back to string
+ */
+function serializeFrontmatter(frontmatter: Record<string, string>): string {
+  const lines = Object.entries(frontmatter).map(([key, value]) => `${key}: ${value}`)
+  return `---\n${lines.join("\n")}\n---\n`
+}
+
+/**
+ * Read a markdown file and parse its content
+ */
+function readMarkdownFile(filePath: string): MarkdownFile | null {
+  try {
+    const content = fs.readFileSync(filePath, "utf-8")
+    const { frontmatter, body } = parseFrontmatter(content)
+    const title = frontmatter.title || path.basename(filePath, ".md")
+
+    return {
+      filePath,
+      title,
+      content: body.trim(),
+      hyperspellId: frontmatter.hyperspell_id || null,
+    }
+  } catch (err) {
+    log.error(`Failed to read markdown file: ${filePath}`, err)
+    return null
+  }
+}
+
+/**
+ * Update the hyperspell_id in the frontmatter of a markdown file
+ */
+function updateFrontmatterId(filePath: string, hyperspellId: string): void {
+  try {
+    const content = fs.readFileSync(filePath, "utf-8")
+    const { frontmatter, body } = parseFrontmatter(content)
+
+    frontmatter.hyperspell_id = hyperspellId
+
+    const newContent = serializeFrontmatter(frontmatter) + body
+    fs.writeFileSync(filePath, newContent)
+
+    log.debug(`Updated frontmatter in ${filePath} with hyperspell_id: ${hyperspellId}`)
+  } catch (err) {
+    log.error(`Failed to update frontmatter in ${filePath}`, err)
+  }
+}
+
+/**
+ * Get all markdown files from the memory directory, including subdirectories
+ */
+export function getMemoryFiles(workspaceDir: string): string[] {
+  const memoryDir = path.join(workspaceDir, "memory")
+
+  if (!fs.existsSync(memoryDir)) {
+    return []
+  }
+
+  const results: string[] = []
+
+  function walk(dir: string): void {
+    try {
+      for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+        const fullPath = path.join(dir, entry.name)
+        if (entry.isDirectory()) {
+          walk(fullPath)
+        } else if (entry.name.endsWith(".md")) {
+          results.push(fullPath)
+        }
+      }
+    } catch (err) {
+      log.error(`Failed to read directory: ${dir}`, err)
+    }
+  }
+
+  walk(memoryDir)
+  return results
+}
+
+/**
+ * Sync a single markdown file to Hyperspell
+ */
+export async function syncMarkdownFile(
+  client: HyperspellClient,
+  filePath: string,
+): Promise<{ success: boolean; resourceId?: string; error?: string }> {
+  const file = readMarkdownFile(filePath)
+  if (!file) {
+    return { success: false, error: "Failed to read file" }
+  }
+
+  if (!file.content) {
+    return { success: false, error: "File has no content" }
+  }
+
+  try {
+    const result = await client.addMemory(file.content, {
+      title: file.title,
+      resourceId: file.hyperspellId || undefined,
+      collection: "openclaw",
+      metadata: {
+        openclaw_source: "memory_sync",
+        file_path: filePath,
+      },
+    })
+
+    // Update frontmatter with new resource ID if it changed or was newly created
+    if (result.resourceId !== file.hyperspellId) {
+      updateFrontmatterId(filePath, result.resourceId)
+    }
+
+    return { success: true, resourceId: result.resourceId }
+  } catch (err) {
+    const errorMsg = err instanceof Error ? err.message : String(err)
+    log.error(`Failed to sync ${filePath}`, err)
+    return { success: false, error: errorMsg }
+  }
+}
+
+/**
+ * Sync all markdown files in the memory directory
+ */
+export async function syncAllMemoryFiles(
+  client: HyperspellClient,
+  workspaceDir: string,
+): Promise<{ synced: number; failed: number; errors: string[] }> {
+  const files = getMemoryFiles(workspaceDir)
+  let synced = 0
+  let failed = 0
+  const errors: string[] = []
+
+  for (const filePath of files) {
+    const result = await syncMarkdownFile(client, filePath)
+    if (result.success) {
+      synced++
+      log.info(`Synced: ${path.basename(filePath)} -> ${result.resourceId}`)
+    } else {
+      failed++
+      errors.push(`${path.basename(filePath)}: ${result.error}`)
+    }
+  }
+
+  return { synced, failed, errors }
+}