@m16khb/llm-wiki 0.1.0

package/internal/wiki/lint.go

@@ -0,0 +1,91 @@
+package wiki
+
+import (
+	"os"
+	"regexp"
+	"sort"
+	"strings"
+)
+
+var wikilinkPattern = regexp.MustCompile(`\[\[([^\]]+)\]\]`)
+
+// LintResult reports structural issues that matter to LLM Wiki agents.
+type LintResult struct {
+	OK                 bool     `json:"ok"`
+	MarkdownFiles      int      `json:"markdown_files"`
+	MissingFrontmatter []string `json:"missing_frontmatter,omitempty"`
+	BrokenWikilinks    []string `json:"broken_wikilinks,omitempty"`
+	DuplicateSlugs     []string `json:"duplicate_slugs,omitempty"`
+	Warnings           []string `json:"warnings,omitempty"`
+	Errors             []string `json:"errors,omitempty"`
+}
+
+func (v *Vault) Lint() (LintResult, error) {
+	pages, err := v.ListPages()
+	if err != nil {
+		return LintResult{}, err
+	}
+	res := LintResult{MarkdownFiles: len(pages)}
+	slugs := map[string][]string{}
+	paths := map[string]bool{}
+	for _, p := range pages {
+		slugs[p.Slug] = append(slugs[p.Slug], p.Path)
+		paths[strings.TrimSuffix(p.Path, ".md")] = true
+		paths[p.Path] = true
+		data, err := os.ReadFile(p.AbsPath)
+		if err != nil {
+			return res, err
+		}
+		if _, _, ok := ParseFrontmatter(string(data)); !ok {
+			res.MissingFrontmatter = append(res.MissingFrontmatter, p.Path)
+		}
+	}
+	for slug, locations := range slugs {
+		if len(locations) > 1 {
+			sort.Strings(locations)
+			res.DuplicateSlugs = append(res.DuplicateSlugs, slug+": "+strings.Join(locations, ", "))
+			res.Warnings = append(res.Warnings, "duplicate slug: "+slug)
+		}
+	}
+	knownSlugs := map[string]bool{}
+	for slug := range slugs {
+		knownSlugs[slug] = true
+	}
+	for _, p := range pages {
+		data, err := os.ReadFile(p.AbsPath)
+		if err != nil {
+			return res, err
+		}
+		for _, match := range wikilinkPattern.FindAllStringSubmatch(string(data), -1) {
+			ref := normalizeWikilink(match[1])
+			if ref == "" || isExternalLike(ref) {
+				continue
+			}
+			if knownSlugs[ref] || paths[ref] || paths[strings.TrimSuffix(ref, ".md")] {
+				continue
+			}
+			res.BrokenWikilinks = append(res.BrokenWikilinks, p.Path+" -> "+ref)
+		}
+	}
+	sort.Strings(res.MissingFrontmatter)
+	sort.Strings(res.BrokenWikilinks)
+	sort.Strings(res.DuplicateSlugs)
+	res.OK = len(res.MissingFrontmatter) == 0 && len(res.BrokenWikilinks) == 0 && len(res.Errors) == 0
+	return res, nil
+}
+
+func normalizeWikilink(raw string) string {
+	raw = strings.TrimSpace(raw)
+	raw = strings.Split(raw, "|")[0]
+	raw = strings.Split(raw, "#")[0]
+	raw = strings.TrimSpace(raw)
+	raw = strings.TrimSuffix(raw, ".md")
+	if strings.Contains(raw, "/") {
+		return strings.Trim(raw, "/")
+	}
+	return raw
+}
+
+func isExternalLike(ref string) bool {
+	return strings.Contains(ref, "://") || strings.HasPrefix(ref, "#")
+}

package/internal/wiki/search.go

@@ -0,0 +1,183 @@
+package wiki
+
+import (
+	"os"
+	"sort"
+	"strings"
+	"unicode/utf8"
+)
+
+// SearchOptions controls vault search.
+type SearchOptions struct {
+	Query          string   `json:"query"`
+	Limit          int      `json:"limit"`
+	Roots          []string `json:"roots,omitempty"`
+	IncludeArchive bool     `json:"include_archive,omitempty"`
+}
+
+// SearchResult is one ranked page hit.
+type SearchResult struct {
+	Slug    string   `json:"slug"`
+	Path    string   `json:"path"`
+	Title   string   `json:"title,omitempty"`
+	Type    string   `json:"type,omitempty"`
+	Domain  string   `json:"domain,omitempty"`
+	Tags    []string `json:"tags,omitempty"`
+	Updated string   `json:"updated,omitempty"`
+	Score   int      `json:"score"`
+	Snippet string   `json:"snippet"`
+}
+
+func (v *Vault) Search(opts SearchOptions) ([]SearchResult, error) {
+	query := strings.TrimSpace(opts.Query)
+	if opts.Limit <= 0 || opts.Limit > 50 {
+		opts.Limit = 10
+	}
+	pages, err := v.ListPages()
+	if err != nil {
+		return nil, err
+	}
+	terms := queryTerms(query)
+	var results []SearchResult
+	for _, p := range pages {
+		if !opts.IncludeArchive && strings.HasPrefix(p.Path, "_archive/") {
+			continue
+		}
+		if !pathInRoots(p.Path, opts.Roots) {
+			continue
+		}
+		data, err := os.ReadFile(p.AbsPath)
+		if err != nil {
+			return nil, err
+		}
+		content := string(data)
+		score := scorePage(p, content, terms)
+		if query != "" && score == 0 {
+			continue
+		}
+		results = append(results, SearchResult{
+			Slug:    p.Slug,
+			Path:    p.Path,
+			Title:   firstNonEmpty(p.Metadata.Title, p.Slug),
+			Type:    p.Metadata.Type,
+			Domain:  p.Metadata.Domain,
+			Tags:    p.Metadata.Tags,
+			Updated: p.Metadata.Updated,
+			Score:   score,
+			Snippet: snippet(content, terms),
+		})
+	}
+	sort.Slice(results, func(i, j int) bool {
+		if results[i].Score == results[j].Score {
+			return results[i].Path < results[j].Path
+		}
+		return results[i].Score > results[j].Score
+	})
+	if len(results) > opts.Limit {
+		results = results[:opts.Limit]
+	}
+	return results, nil
+}
+
+func queryTerms(query string) []string {
+	fields := strings.Fields(strings.ToLower(query))
+	seen := map[string]bool{}
+	out := make([]string, 0, len(fields))
+	for _, f := range fields {
+		f = strings.Trim(f, " \t\n\r.,;:!?()[]{}\"'")
+		if f == "" || seen[f] {
+			continue
+		}
+		seen[f] = true
+		out = append(out, f)
+	}
+	return out
+}
+
+func pathInRoots(path string, roots []string) bool {
+	if len(roots) == 0 {
+		return true
+	}
+	for _, root := range roots {
+		root = strings.Trim(strings.TrimSpace(root), "/")
+		if root == "" {
+			continue
+		}
+		if path == root || strings.HasPrefix(path, root+"/") {
+			return true
+		}
+	}
+	return false
+}
+
+func scorePage(p Page, content string, terms []string) int {
+	if len(terms) == 0 {
+		return 1
+	}
+	hayPath := strings.ToLower(p.Path + " " + p.Slug + " " + p.Metadata.Title + " " + strings.Join(p.Metadata.Tags, " "))
+	hayBody := strings.ToLower(content)
+	score := 0
+	for _, term := range terms {
+		if strings.Contains(hayPath, term) {
+			score += 10
+		}
+		bodyCount := strings.Count(hayBody, term)
+		if bodyCount > 20 {
+			bodyCount = 20
+		}
+		score += bodyCount
+	}
+	return score
+}
+
+func snippet(content string, terms []string) string {
+	_, body, ok := ParseFrontmatter(content)
+	if ok {
+		content = body
+	}
+	clean := strings.Join(strings.Fields(content), " ")
+	if clean == "" {
+		return ""
+	}
+	lower := strings.ToLower(clean)
+	idx := -1
+	for _, term := range terms {
+		if i := strings.Index(lower, term); i >= 0 && (idx < 0 || i < idx) {
+			idx = i
+		}
+	}
+	if idx < 0 {
+		idx = 0
+	}
+	start := idx - 140
+	if start < 0 {
+		start = 0
+	}
+	end := idx + 360
+	if end > len(clean) {
+		end = len(clean)
+	}
+	for start > 0 && !utf8.RuneStart(clean[start]) {
+		start--
+	}
+	for end < len(clean) && !utf8.RuneStart(clean[end]) {
+		end++
+	}
+	out := clean[start:end]
+	if start > 0 {
+		out = "…" + out
+	}
+	if end < len(clean) {
+		out += "…"
+	}
+	return out
+}
+
+func firstNonEmpty(values ...string) string {
+	for _, v := range values {
+		if strings.TrimSpace(v) != "" {
+			return v
+		}
+	}
+	return ""
+}

package/internal/wiki/vault.go

@@ -0,0 +1,279 @@
+package wiki
+
+import (
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"time"
+)
+
+const DefaultVaultRelative = "workspace/knowledge-base/llm-wiki"
+
+// Vault provides safe filesystem access to an Obsidian-backed LLM Wiki vault.
+type Vault struct {
+	Root string
+	Now  func() time.Time
+}
+
+// Page is a Markdown page discovered in the vault.
+type Page struct {
+	Slug     string   `json:"slug"`
+	Path     string   `json:"path"`
+	AbsPath  string   `json:"-"`
+	Metadata Metadata `json:"metadata"`
+	Size     int64    `json:"size"`
+	ModTime  string   `json:"modified"`
+}
+
+// Info summarizes vault state.
+type Info struct {
+	Root              string         `json:"root"`
+	MarkdownFiles     int            `json:"markdown_files"`
+	Sources           int            `json:"sources"`
+	RawSnapshotSource int            `json:"raw_snapshot_sources"`
+	Concepts          int            `json:"concepts"`
+	Entities          int            `json:"entities"`
+	Sessions          int            `json:"sessions"`
+	ByType            map[string]int `json:"by_type"`
+}
+
+func New(root string) (*Vault, error) {
+	if root == "" {
+		root = DefaultRoot()
+	}
+	abs, err := filepath.Abs(expandHome(root))
+	if err != nil {
+		return nil, err
+	}
+	st, err := os.Stat(abs)
+	if err != nil {
+		return nil, fmt.Errorf("vault root not found: %w", err)
+	}
+	if !st.IsDir() {
+		return nil, fmt.Errorf("vault root is not a directory: %s", abs)
+	}
+	return &Vault{Root: abs, Now: time.Now}, nil
+}
+
+func DefaultRoot() string {
+	if v := os.Getenv("LLM_WIKI_ROOT"); v != "" {
+		return v
+	}
+	if v := os.Getenv("LLM_WIKI_VAULT"); v != "" {
+		return v
+	}
+	home, err := os.UserHomeDir()
+	if err != nil || home == "" {
+		return "."
+	}
+	candidates := []string{
+		filepath.Join(home, DefaultVaultRelative),
+		filepath.Join(home, "Workspace", "knowledge-base", "llm-wiki"),
+	}
+	for _, c := range candidates {
+		if st, err := os.Stat(c); err == nil && st.IsDir() {
+			return c
+		}
+	}
+	return candidates[0]
+}
+
+func expandHome(path string) string {
+	if path == "~" {
+		if home, err := os.UserHomeDir(); err == nil {
+			return home
+		}
+	}
+	if strings.HasPrefix(path, "~/") {
+		if home, err := os.UserHomeDir(); err == nil {
+			return filepath.Join(home, path[2:])
+		}
+	}
+	return path
+}
+
+func (v *Vault) SafeJoin(rel string) (string, error) {
+	if strings.TrimSpace(rel) == "" {
+		return "", errors.New("path is empty")
+	}
+	if filepath.IsAbs(rel) {
+		return "", fmt.Errorf("absolute paths are not allowed: %s", rel)
+	}
+	rel = filepath.Clean(filepath.FromSlash(rel))
+	if rel == "." || strings.HasPrefix(rel, "..") || strings.Contains(rel, string(filepath.Separator)+".."+string(filepath.Separator)) {
+		return "", fmt.Errorf("path escapes vault: %s", rel)
+	}
+	if rel == ".obsidian" || strings.HasPrefix(rel, ".obsidian"+string(filepath.Separator)) {
+		return "", errors.New(".obsidian is not accessible through llm-wiki")
+	}
+	abs := filepath.Join(v.Root, rel)
+	rootWithSep := v.Root + string(filepath.Separator)
+	if abs != v.Root && !strings.HasPrefix(abs, rootWithSep) {
+		return "", fmt.Errorf("path escapes vault: %s", rel)
+	}
+	return abs, nil
+}
+
+func (v *Vault) Rel(abs string) string {
+	rel, err := filepath.Rel(v.Root, abs)
+	if err != nil {
+		return abs
+	}
+	return filepath.ToSlash(rel)
+}
+
+func (v *Vault) ListPages() ([]Page, error) {
+	var pages []Page
+	err := filepath.WalkDir(v.Root, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		name := d.Name()
+		if d.IsDir() {
+			if path == v.Root {
+				return nil
+			}
+			if shouldSkipDir(name) {
+				return filepath.SkipDir
+			}
+			return nil
+		}
+		if !strings.EqualFold(filepath.Ext(name), ".md") {
+			return nil
+		}
+		page, err := v.ReadPageByAbs(path)
+		if err != nil {
+			return err
+		}
+		pages = append(pages, page)
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	sort.Slice(pages, func(i, j int) bool { return pages[i].Path < pages[j].Path })
+	return pages, nil
+}
+
+func shouldSkipDir(name string) bool {
+	switch name {
+	case ".git", ".obsidian", ".omx", ".omc", ".claude", "node_modules", "vendor", "__pycache__":
+		return true
+	}
+	return strings.HasPrefix(name, ".")
+}
+
+func (v *Vault) ReadPageByAbs(abs string) (Page, error) {
+	data, err := os.ReadFile(abs)
+	if err != nil {
+		return Page{}, err
+	}
+	st, err := os.Stat(abs)
+	if err != nil {
+		return Page{}, err
+	}
+	meta, _, _ := ParseFrontmatter(string(data))
+	rel := v.Rel(abs)
+	slug := strings.TrimSuffix(filepath.Base(rel), filepath.Ext(rel))
+	return Page{Slug: slug, Path: rel, AbsPath: abs, Metadata: meta, Size: st.Size(), ModTime: st.ModTime().Format(time.RFC3339)}, nil
+}
+
+func (v *Vault) ReadPage(ref string, maxBytes int) (Page, string, error) {
+	abs, err := v.ResolvePage(ref)
+	if err != nil {
+		return Page{}, "", err
+	}
+	data, err := os.ReadFile(abs)
+	if err != nil {
+		return Page{}, "", err
+	}
+	if maxBytes > 0 && len(data) > maxBytes {
+		data = append(data[:maxBytes], []byte("\n\n[llm-wiki: truncated]\n")...)
+	}
+	page, err := v.ReadPageByAbs(abs)
+	if err != nil {
+		return Page{}, "", err
+	}
+	return page, string(data), nil
+}
+
+func (v *Vault) ResolvePage(ref string) (string, error) {
+	ref = strings.TrimSpace(ref)
+	ref = strings.Trim(ref, "[]")
+	ref = strings.Split(ref, "#")[0]
+	ref = strings.Split(ref, "|")[0]
+	ref = strings.TrimSpace(ref)
+	if ref == "" {
+		return "", errors.New("page reference is empty")
+	}
+	if strings.HasSuffix(ref, ".md") || strings.Contains(ref, "/") {
+		abs, err := v.SafeJoin(ref)
+		if err != nil {
+			return "", err
+		}
+		if _, err := os.Stat(abs); err == nil {
+			return abs, nil
+		}
+	}
+	slug := strings.TrimSuffix(filepath.Base(ref), ".md")
+	var matches []string
+	err := filepath.WalkDir(v.Root, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if d.IsDir() {
+			if path != v.Root && shouldSkipDir(d.Name()) {
+				return filepath.SkipDir
+			}
+			return nil
+		}
+		if strings.EqualFold(strings.TrimSuffix(d.Name(), filepath.Ext(d.Name())), slug) && strings.EqualFold(filepath.Ext(d.Name()), ".md") {
+			matches = append(matches, path)
+		}
+		return nil
+	})
+	if err != nil {
+		return "", err
+	}
+	if len(matches) == 0 {
+		return "", fmt.Errorf("page not found: %s", ref)
+	}
+	sort.Strings(matches)
+	return matches[0], nil
+}
+
+func (v *Vault) Info() (Info, error) {
+	pages, err := v.ListPages()
+	if err != nil {
+		return Info{}, err
+	}
+	info := Info{Root: v.Root, MarkdownFiles: len(pages), ByType: map[string]int{}}
+	for _, p := range pages {
+		t := p.Metadata.Type
+		if t == "" {
+			t = "unknown"
+		}
+		info.ByType[t]++
+		if strings.HasPrefix(p.Path, "10-sources/") {
+			info.Sources++
+			content, _ := os.ReadFile(p.AbsPath)
+			if strings.Contains(string(content), "canonical_source: snapshot") || strings.Contains(string(content), "fidelity: raw-snapshot") {
+				info.RawSnapshotSource++
+			}
+		}
+		if strings.HasPrefix(p.Path, "20-wiki/concepts/") {
+			info.Concepts++
+		}
+		if strings.HasPrefix(p.Path, "20-wiki/entities/") {
+			info.Entities++
+		}
+		if strings.HasPrefix(p.Path, "30-sessions/") {
+			info.Sessions++
+		}
+	}
+	return info, nil
+}