@jerryjiao/knowflow 0.3.0

package/scripts/graph_relation_labeler.py

@@ -0,0 +1,191 @@
+#!/usr/bin/env python3
+"""
+KnowFlow Graph Relation Labeler
+Infers semantic relation types from wikilink context and updates graph.json
+"""
+
+import argparse
+import re, json, os
+from pathlib import Path
+from collections import defaultdict
+
+PROJECT_ROOT = Path(__file__).resolve().parent.parent
+DEFAULT_WIKI_DIR = os.environ.get("KNOWFLOW_WIKI_DIR", str(PROJECT_ROOT / "wiki"))
+DEFAULT_GRAPH_OUTPUT = Path(os.environ.get("KNOWFLOW_GRAPH_OUTPUT", PROJECT_ROOT / "graph" / "graph.html"))
+DEFAULT_GRAPH_JSON = os.environ.get("KNOWFLOW_GRAPH_JSON", str(DEFAULT_GRAPH_OUTPUT.with_suffix(".json")))
+
+# ── Relation inference rules ──────────────────────────────
+
+# Category-based default relations
+CATEGORY_RELATIONS = {
+    # source → entity: "cites" or "describes"
+    ("sources", "entities"): "描述",
+    ("sources", "concepts"): "阐述",
+    ("sources", "comparisons"): "参考",
+    ("sources", "sources"): "相关",
+    # entity → concept: "implements" or "exemplifies"
+    ("entities", "concepts"): "体现",
+    ("entities", "entities"): "关联",
+    ("entities", "sources"): "来源",
+    # concept → entity: "applies_to" or "governs"
+    ("concepts", "entities"): "适用于",
+    ("concepts", "concepts"): "相关",
+    ("concepts", "sources"): "引用",
+    # comparison → others
+    ("comparisons", "entities"): "对比",
+    ("comparisons", "concepts"): "评估",
+    ("comparisons", "sources"): "依据",
+}
+
+def detect_category(filepath):
+    parts = Path(filepath).parts
+    if len(parts) >= 2:
+        parent = parts[-2]
+        if parent in ["sources", "entities", "concepts", "comparisons"]:
+            return parent
+    stem = Path(filepath).stem.lower()
+    if stem in ("index", "log", "overview"):
+        return "_special"
+    return "sources"
+
+# Context-based relation refinement
+CONTEXT_PATTERNS = [
+    (r'(来源|来自|基于|引自|参考|参见)', "来源"),
+    (r'(相关|关联|类似|同类|同属)', "关联"),
+    (r'(对比|比较|vs\.?|versus)', "对比"),
+    (r'(实现|使用|采用|应用|工具)', "使用"),
+    (r'(属于|归类|分类|类型)', "属于"),
+    (r'(替代|取代|替换)', "替代"),
+    (r'(衍生|扩展|进化|发展)', "衍生"),
+    (r'(姊妹|兄弟|姐妹项目)', "姊妹"),
+    (r'(推荐|首选|建议)', "推荐"),
+    (r'(创建者|作者|开发者|创始人)', "创建"),
+    (r'(支持|兼容|集成|接入)', "支持"),
+    (r'(竞争对手|竞品|对手)', "竞争"),
+]
+
+def infer_relation(from_path, to_path, from_content, all_categories):
+    """Infer the most appropriate relation label for an edge."""
+    from_cat = detect_category(from_path)
+    to_cat = detect_category(to_path)
+    
+    # 1. Try category-based default
+    default = CATEGORY_RELATIONS.get((from_cat, to_cat))
+    
+    # 2. Refine based on context around the wikilink
+    link_target = Path(to_path).stem
+    
+    # Find the wikilink in content and check surrounding text
+    pattern = r'\[\[' + re.escape(link_target) + r'([^\]]*)\]\]'
+    matches = list(re.finditer(pattern, from_content))
+    
+    if matches:
+        # Check context around each match (50 chars before/after)
+        for m in matches:
+            start = max(0, m.start() - 50)
+            end = min(len(from_content), m.end() + 50)
+            context = from_content[start:end]
+            for pat, rel in CONTEXT_PATTERNS:
+                if re.search(pat, context):
+                    return rel
+    
+    # 3. Fall back to category default or generic
+    return default or "链接"
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Add semantic labels to a KnowFlow graph")
+    parser.add_argument("--wiki-dir", default=DEFAULT_WIKI_DIR, help="Wiki directory")
+    parser.add_argument("--graph-json", default=DEFAULT_GRAPH_JSON, help="Graph JSON file")
+    args = parser.parse_args()
+    wiki_dir = Path(args.wiki_dir).resolve()
+    graph_json = Path(args.graph_json).resolve()
+
+    print("🏷️  开始为图谱边添加关系标签...")
+    
+    # Load existing graph
+    with graph_json.open('r', encoding='utf-8') as f:
+        graph = json.load(f)
+    
+    # Read all wiki contents for context analysis
+    wiki_path = wiki_dir
+    contents = {}
+    for md_file in wiki_path.rglob('*.md'):
+        rel = str(md_file.relative_to(wiki_path))
+        try:
+            contents[rel] = md_file.read_text(encoding='utf-8')
+        except:
+            pass
+    
+    # Build category cache
+    categories = {p: detect_category(p) for p in contents}
+    
+    # Process each edge
+    updated = 0
+    relation_counts = defaultdict(int)
+    
+    for edge in graph.get('edges', []):
+        from_id = edge.get('from', '')
+        to_id = edge.get('to', '')
+        
+        if not from_id or not to_id:
+            continue
+        
+        from_content = contents.get(from_id, '')
+        relation = infer_relation(from_id, to_id, from_content, categories)
+        
+        edge['relation'] = relation
+        edge['title'] = f"{relation}"  # tooltip
+        relation_counts[relation] += 1
+        updated += 1
+    
+    # Also add relation info to node titles for better tooltips
+    for node in graph.get('nodes', []):
+        nid = node.get('id', '')
+        cat = categories.get(nid, detect_category(nid))
+        # Add Chinese category label
+        cat_labels = {
+            'sources': '📥 来源', 'entities': '👤 实体',
+            'concepts': '💡 概念', 'comparisons': '⚖️ 对比',
+            '_special': '⭐ 索引'
+        }
+        existing_title = node.get('title', '')
+        if cat_labels.get(cat) and cat_labels[cat] not in existing_title:
+            node['category_label'] = cat_labels[cat]
+    
+    # Save updated graph
+    with graph_json.open('w', encoding='utf-8') as f:
+        json.dump(graph, f, ensure_ascii=False, indent=2)
+    
+    print(f"✅ 完成！更新了 {updated} 条边的关系标签")
+    print(f"\n📊 关系分布:")
+    for rel, count in sorted(relation_counts.items(), key=lambda x: -x[1]):
+        bar = "█" * min(count, 40)
+        print(f"  {rel:8s} {count:4d}  {bar}")
+    
+    # Stats
+    total_edges = len(graph.get('edges', []))
+    total_nodes = len(graph.get('nodes', []))
+    
+    # Count isolated nodes (degree 0)
+    connected = set()
+    for e in graph.get('edges', []):
+        connected.add(e.get('from'))
+        connected.add(e.get('to'))
+    isolated = [n for n in graph.get('nodes', []) if n.get('id') not in connected]
+    
+    print(f"\n📈 图谱统计:")
+    print(f"  总节点: {total_nodes}")
+    print(f"  总边数: {total_edges}")
+    print(f"  连接节点: {len(connected)}")
+    print(f"  孤立节点: {len(isolated)} ({len(isolated)*100//max(total_nodes,1)}%)")
+    
+    if isolated:
+        print(f"\n⚠️  孤立节点列表:")
+        for n in isolated[:10]:
+            print(f"  - {n.get('label', n.get('id'))} ({n.get('id')})")
+        if len(isolated) > 10:
+            print(f"  ... 还有 {len(isolated)-10} 个")
+
+if __name__ == '__main__':
+    main()

package/scripts/ingest.sh

@@ -0,0 +1,162 @@
+#!/bin/bash
+# knowflow ingest pipeline — 自动识别来源并提取全文
+# Usage: bash ingest.sh <url_or_content> [source_type]
+# Source type: auto | twitter | xiaohongshu | wechat | web | youtube | text
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+WIKI_ROOT="${KNOWFLOW_ROOT:-$(cd "$SCRIPT_DIR/.." && pwd)}"
+RAW_DIR="${KNOWFLOW_RAW_DIR:-$WIKI_ROOT/raw}"
+TIMESTAMP=$(date +%Y-%m-%d-%H%M)
+
+URL="${1:-}"
+SOURCE_TYPE="${2:-auto}"
+
+if [ -z "$URL" ]; then
+  echo "❌ Usage: bash ingest.sh <url_or_text> [source_type]"
+  exit 1
+fi
+
+# Auto-detect source type from URL
+detect_source() {
+  local url="$1"
+  if echo "$url" | grep -qi 'x\.com\|twitter\.com'; then
+    echo "twitter"
+  elif echo "$url" | grep -qi 'xiaohongshu\|xhslink\|xhscdn'; then
+    echo "xiaohongshu"
+  elif echo "$url" | grep -qi 'mp\.weixin\|weixin.*article\|wx'; then
+    echo "wechat"
+  elif echo "$url" | grep -qi 'youtube\|youtu\.be'; then
+    echo "youtube"
+  elif echo "$url" | grep -qi '^https\?://'; then
+    echo "web"
+  else
+    echo "text"
+  fi
+}
+
+[ "$SOURCE_TYPE" = "auto" ] && SOURCE_TYPE=$(detect_source "$URL")
+
+# ── URL Sanitization ─────────────────────────────────────
+validate_url() {
+  local url="$1"
+  # Arguments are passed without a shell; keep normal query strings intact.
+  if ! echo "$url" | grep -qE '^https?://[^[:space:]]+$'; then
+    echo "❌ Invalid URL: $url" >&2
+    return 1
+  fi
+}
+if [ "$SOURCE_TYPE" != "text" ]; then
+  validate_url "$URL" || exit 1
+fi
+
+mkdir -p "$RAW_DIR"/{twitter,xiaohongshu,wechat,web}
+
+OUTPUT_FILE=""
+EXTRACT_METHOD=""
+
+case "$SOURCE_TYPE" in
+  twitter)
+    echo "🐦 检测到 Twitter/X 链接..."
+    OUTPUT_FILE="$RAW_DIR/twitter/$TIMESTAMP-tweet.md"
+    # Extract tweet via twitter CLI or Jina
+    if command -v twitter &>/dev/null; then
+      TWEET_ID=$(echo "$URL" | grep -oE '[0-9]{15,}' | head -1)
+      if [ -n "$TWEET_ID" ]; then
+        twitter get "$TWEET_ID" 2>/dev/null > "$OUTPUT_FILE" || \
+          curl -sL "https://r.jina.ai/$URL" > "$OUTPUT_FILE" 2>/dev/null
+      else
+        curl -sL "https://r.jina.ai/$URL" > "$OUTPUT_FILE" 2>/dev/null
+      fi
+    else
+      curl -sL "https://r.jina.ai/$URL" > "$OUTPUT_FILE" 2>/dev/null
+    fi
+    EXTRACT_METHOD="jina_reader"
+    ;;
+
+  xiaohongshu)
+    echo "📕 检测到小红书链接..."
+    OUTPUT_FILE="$RAW_DIR/xiaohongshu/$TIMESTAMP-xhs.md"
+    # Use agent-browser for xiaohongshu (needs JS rendering)
+    if command -v agent-browser &>/dev/null; then
+      # Try Jina first, fallback to agent-browser
+      HTTP_CODE=$(curl -sL -o "$OUTPUT_FILE" -w "%{http_code}" --max-time 15 "https://r.jina.ai/$URL" 2>/dev/null) || true
+      if [ "$HTTP_CODE" != "200" ] || [ ! -s "$OUTPUT_FILE" ]; then
+        echo "> ⚠️ Jina 提取失败，小红书可能需要浏览器渲染，已保存原始链接" > "$OUTPUT_FILE"
+        echo "" >> "$OUTPUT_FILE"
+        echo "**原始链接**: $URL" >> "$OUTPUT_FILE"
+        echo "" >> "$OUTPUT_FILE"
+        echo "> 💡 提示: 小红书内容需要通过 agent-browser 渲染提取" >> "$OUTPUT_FILE"
+      fi
+    else
+      curl -sL "https://r.jina.ai/$URL" > "$OUTPUT_FILE" 2>/dev/null || echo "**原始链接**: $URL" > "$OUTPUT_FILE"
+    fi
+    EXTRACT_METHOD="jina_reader"
+    ;;
+
+  wechat)
+    echo "💬 检测到微信公众号链接..."
+    OUTPUT_FILE="$RAW_DIR/wechat/$TIMESTAMP-wechat.md"
+    curl -sL --max-time 20 "https://r.jina.ai/$URL" > "$OUTPUT_FILE" 2>/dev/null
+    EXTRACT_METHOD="jina_reader"
+    ;;
+
+  youtube)
+    echo "▶️ 检测到 YouTube 链接..."
+    OUTPUT_FILE="$RAW_DIR/web/$TIMESTAMP-youtube.md"
+    if command -v yt-dlp &>/dev/null; then
+      # Get video info + transcript
+      echo "# YouTube Video" > "$OUTPUT_FILE"
+      echo "" >> "$OUTPUT_FILE"
+      echo "**URL**: $URL" >> "$OUTPUT_FILE"
+      echo "" >> "$OUTPUT_FILE"
+      echo "## Video Info" >> "$OUTPUT_FILE"
+      yt-dlp --print title --print description --print duration_string --no-download "$URL" >> "$OUTPUT_FILE" 2>/dev/null || true
+      echo "" >> "$OUTPUT_FILE"
+      echo "## Transcript / Subtitles" >> "$OUTPUT_FILE"
+      yt-dlp --write-sub --sub-langs "zh,en" --skip-download -o "/tmp/wiki-yt-sub" "$URL" 2>/dev/null && \
+        cat /tmp/wiki-yt-sub*.vtt 2>/dev/null | sed 's/<[^>]*>//g' | grep -v '^$' | head -200 >> "$OUTPUT_FILE" || \
+        echo "(无字幕可用)" >> "$OUTPUT_FILE"
+      rm -f /tmp/wiki-yt-sub* 2>/dev/null || true
+    else
+      curl -sL "https://r.jina.ai/$URL" > "$OUTPUT_FILE" 2>/dev/null
+    fi
+    EXTRACT_METHOD="yt-dlp+jina"
+    ;;
+
+  web)
+    echo "🌐 检测到网页链接..."
+    # Determine subdirectory by domain
+    DOMAIN=$(echo "$URL" | sed 's|https\?://||' | cut -d'/' -f1)
+    OUTPUT_FILE="$RAW_DIR/web/$TIMESTAMP-web-${DOMAIN%%.*}.md"
+    curl -sL --max-time 20 "https://r.jina.ai/$URL" > "$OUTPUT_FILE" 2>/dev/null
+    EXTRACT_METHOD="jina_reader"
+    ;;
+
+  text)
+    echo "📝 纯文本内容..."
+    OUTPUT_FILE="$RAW_DIR/web/$TIMESTAMP-text.md"
+    echo "$URL" > "$OUTPUT_FILE"
+    EXTRACT_METHOD="direct"
+    ;;
+esac
+
+# Verify output
+if [ -f "$OUTPUT_FILE" ] && [ -s "$OUTPUT_FILE" ]; then
+  SIZE=$(wc -c < "$OUTPUT_FILE" | tr -d ' ')
+  LINES=$(wc -l < "$OUTPUT_FILE" | tr -d ' ')
+  echo ""
+  echo "✅ 提取完成:"
+  echo "   📄 文件: $OUTPUT_FILE"
+  echo "   📏 大小: ${SIZE} bytes (${LINES} 行)"
+  echo "   🔧 方法: $EXTRACT_METHOD"
+  echo "   🏷️ 类型: $SOURCE_TYPE"
+  echo ""
+  echo "→ 下一步: 告诉 Agent 执行 ingest 处理此文件"
+else
+  echo ""
+  echo "⚠️ 提取可能失败（文件为空或不存在）"
+  echo "   原始链接: $URL"
+  exit 1
+fi

package/scripts/pipeline.sh

@@ -0,0 +1,73 @@
+#!/bin/bash
+# knowflow pipeline — automated 5-step wiki processing pipeline
+# Usage: bash scripts/pipeline.sh [--dry-run] [--step=N]
+#
+# Steps:
+#   1. bookmark_sync.sh     — sync X/Twitter bookmarks
+#   2. detect new raw/ files — compare .ingest-state.json vs raw/
+#   3. ingest new files      — run ingest.sh for each new file
+#   4. vector-store build    — incremental vector index (only if new pages)
+#   5. wiki-health.sh        — broken links, empty files, orphan pages
+#
+# Options:
+#   --dry-run    Print each step without executing
+#   --step=N     Run only step N (1-5)
+
+set -euo pipefail
+
+# ── Temp file cleanup ─────────────────────────────────────
+_TEMP_FILES=()
+cleanup() { rm -f "${_TEMP_FILES[@]:-}" 2>/dev/null || true; }
+trap cleanup EXIT INT TERM
+
+# ── Config ──────────────────────────────────────────────
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+WIKI_ROOT="${KNOWFLOW_ROOT:-${WIKI_ROOT:-$(cd "$SCRIPT_DIR/.." && pwd)}}"
+STATE_FILE="$WIKI_ROOT/.ingest-state.json"
+RAW_DIR="${KNOWFLOW_RAW_DIR:-$WIKI_ROOT/raw}"
+WIKI_DIR="${KNOWFLOW_WIKI_DIR:-$WIKI_ROOT/wiki}"
+LOCK_FILE="$WIKI_ROOT/.pipeline.lock"
+TIMESTAMP=$(date +%Y-%m-%d-%H%M)
+
+DRY_RUN=false
+RUN_STEP=0
+
+# ── Parse args ──────────────────────────────────────────
+for arg in "$@"; do
+  case "$arg" in
+    --dry-run)   DRY_RUN=true ;;
+    --step=*)    RUN_STEP="${arg#--step=}" ;;
+    *)           echo "Unknown option: $arg" >&2; exit 1 ;;
+  esac
+done
+
+# ── Helpers ─────────────────────────────────────────────
+step_header() {
+  local step_num="$1"
+  local step_name="$2"
+  echo ""
+  echo "━━━ Step $step_num: $step_name ━━━━━━━━━━━━━━━━━━━━━━━"
+}
+
+run_cmd() {
+  if $DRY_RUN; then
+    echo "[DRY RUN] $*"
+  else
+    "$@"
+  fi
+}
+
+should_run() {
+  local step="$1"
+  [ "$RUN_STEP" -eq 0 ] || [ "$RUN_STEP" -eq "$step" ]
+}
+
+# ── Summary tracking ────────────────────────────────────
+STEP_RESULTS=()
+HEALTH_EXIT=0
+HEALTH_SCORE="UNKNOWN"
+NEW_FILES_COUNT=0
+
+if [ "${HEALTH_EXIT:-0}" -ne 0 ] && ! ${DRY_RUN:-false}; then
+  exit 1
+fi

package/scripts/tags-builder.mjs

@@ -0,0 +1,75 @@
+#!/usr/bin/env node
+/**
+ * knowflow tags — build tag hub pages from [[tag/<name>]] links.
+ *
+ * Scans every Markdown page in the wiki for `[[tag/<name>]]` links and
+ * (re)generates `tag/<name>.md` hub pages that link back to every page
+ * carrying the tag. Hub pages older tags that no longer appear anywhere
+ * are removed, so re-running the command is fully idempotent.
+ *
+ * Environment:
+ *   KNOWFLOW_WIKI_DIR — wiki root directory (set by the CLI)
+ */
+import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+
+const WIKI_DIR = process.env.KNOWFLOW_WIKI_DIR;
+if (!WIKI_DIR || !existsSync(WIKI_DIR)) {
+  console.error('knowflow tags: KNOWFLOW_WIKI_DIR is not set or missing');
+  process.exit(1);
+}
+
+const TAG_DIR = join(WIKI_DIR, 'tag');
+const TAG_LINK = /\[\[tag\/([^\]|]+)(?:\|[^\]]*)?\]\]/g;
+
+function* walkMd(dir) {
+  for (const entry of readdirSync(dir, { withFileTypes: true })) {
+    if (entry.name.startsWith('.')) continue;
+    const path = join(dir, entry.name);
+    if (entry.isDirectory()) yield* walkMd(path);
+    else if (entry.name.endsWith('.md')) yield path;
+  }
+}
+
+const tagMap = new Map();
+for (const file of walkMd(WIKI_DIR)) {
+  const rel = file.slice(WIKI_DIR.length + 1);
+  if (rel.startsWith('tag/')) continue;
+  const text = readFileSync(file, 'utf8');
+  for (const match of text.matchAll(TAG_LINK)) {
+    const tag = match[1].trim();
+    if (!tag) continue;
+    if (!tagMap.has(tag)) tagMap.set(tag, new Set());
+    tagMap.get(tag).add(rel.replace(/\.md$/, ''));
+  }
+}
+
+mkdirSync(TAG_DIR, { recursive: true });
+
+let written = 0;
+for (const [tag, pages] of [...tagMap.entries()].sort((a, b) => b[1].size - a[1].size)) {
+  const pageList = [...pages].sort();
+  const content = [
+    `# Tag: ${tag}`,
+    '',
+    `> Auto-generated by \`knowflow tags\` — index of pages linking to [[tag/${tag}]]. Re-run the command to refresh.`,
+    '',
+    `## Pages (${pageList.length})`,
+    '',
+    ...pageList.map(page => `- [[${page}]]`),
+    '',
+  ].join('\n');
+  writeFileSync(join(TAG_DIR, `${tag}.md`), content, 'utf8');
+  written += 1;
+}
+
+let removed = 0;
+for (const entry of existsSync(TAG_DIR) ? readdirSync(TAG_DIR) : []) {
+  if (!entry.endsWith('.md')) continue;
+  if (!tagMap.has(entry.replace(/\.md$/, ''))) {
+    rmSync(join(TAG_DIR, entry));
+    removed += 1;
+  }
+}
+
+console.log(`tags: ${written} hub page(s) under tag/ (${removed} stale removed)`);