wp2txt 2.1.2 → 2.3.0

data/spec/metadata_index_spec.rb

@@ -0,0 +1,208 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+require "tmpdir"
+require "fileutils"
+require "open3"
+require_relative "support/multistream_fixture"
+require_relative "../lib/wp2txt/metadata_index"
+require_relative "../lib/wp2txt/multistream"
+require_relative "../lib/wp2txt/cli"
+
+RSpec.describe "Wp2txt Metadata Index" do
+  include MultistreamFixture
+
+  describe Wp2txt::MetadataIndex do
+    describe ".normalize_category" do
+      it "replaces underscores, trims, and capitalizes the first letter" do
+        expect(described_class.normalize_category(" japanese_films ")).to eq("Japanese films")
+      end
+
+      it "leaves non-ASCII names unchanged" do
+        expect(described_class.normalize_category("日本の映画作品")).to eq("日本の映画作品")
+      end
+    end
+
+    describe ".clean_heading" do
+      it "strips bold markup and resolves links" do
+        expect(described_class.clean_heading("'''Bold''' [[link|Label]]")).to eq("Bold Label")
+      end
+
+      it "strips HTML tags" do
+        expect(described_class.clean_heading("<small>Notes</small>")).to eq("Notes")
+      end
+    end
+
+    describe ".expand_section_names" do
+      it "expands a canonical name to its alias group" do
+        expect(described_class.expand_section_names("Plot")).to include("Plot", "Synopsis")
+      end
+
+      it "expands an alias back to the full group (bidirectional)" do
+        expect(described_class.expand_section_names("Synopsis")).to include("Plot", "Synopsis")
+      end
+
+      it "returns the name itself when no alias group matches" do
+        expect(described_class.expand_section_names("Nonexistent Section")).to eq(["Nonexistent Section"])
+      end
+    end
+
+    describe ".path_for" do
+      it "builds a cache path keyed to the dump file" do
+        path = described_class.path_for("/dumps/jawiki-20260101-pages-articles-multistream.xml.bz2", cache_dir: "/cache")
+        expect(path).to start_with("/cache/jawiki-20260101-pages-articles-multistream")
+        expect(path).to end_with("_meta.sqlite3")
+      end
+    end
+  end
+
+  describe "build and query" do
+    around do |example|
+      Dir.mktmpdir do |dir|
+        @dir = dir
+        @multistream_path, @index_path = create_fixture(dir)
+        @db_path = File.join(dir, "meta.sqlite3")
+        ms_index = Wp2txt::MultistreamIndex.new(@index_path, use_cache: false, show_progress: false)
+        builder = Wp2txt::MetadataIndexBuilder.new(
+          @multistream_path, ms_index.stream_offsets,
+          db_path: @db_path, num_processes: 0
+        )
+        @index = builder.build
+        example.run
+        @index.close
+      end
+    end
+
+    it "records all pages and counts articles excluding redirects and category pages" do
+      stats = @index.stats
+      expect(stats[:page_count]).to eq(8)
+      expect(stats[:article_count]).to eq(3)
+      expect(stats[:dump_name]).to eq("testwiki-20260101")
+    end
+
+    it "finds articles by exact category" do
+      expect(@index.find_articles(category: "Japanese films")).to eq(["Film A"])
+    end
+
+    it "finds articles through subcategories with depth" do
+      expect(@index.find_articles(category: "Films", depth: 1)).to contain_exactly("Film A", "Film B")
+    end
+
+    it "does not include subcategory members at depth 0" do
+      expect(@index.find_articles(category: "Films", depth: 0)).to be_empty
+    end
+
+    it "matches sections through aliases" do
+      titles = @index.find_articles(category: "Films", depth: 1, has_section: "Plot")
+      expect(titles).to contain_exactly("Film A", "Film B")
+    end
+
+    it "matches sections exactly when aliases are disabled" do
+      titles = @index.find_articles(category: "Films", depth: 1, has_section: "Plot", use_aliases: false)
+      expect(titles).to eq(["Film A"])
+    end
+
+    it "excludes redirects from results" do
+      expect(@index.find_articles(title_match: "Old Film")).to be_empty
+    end
+
+    it "filters by title substring" do
+      expect(@index.find_articles(title_match: "Film")).to contain_exactly("Film A", "Film B")
+    end
+
+    it "applies limit and reports full count separately" do
+      titles = @index.find_articles(title_match: "Film", limit: 1)
+      expect(titles.size).to eq(1)
+      expect(@index.count_articles(title_match: "Film")).to eq(2)
+    end
+
+    it "returns the category tree with depths" do
+      tree = @index.category_tree("Films", depth: 1)
+      expect(tree).to include({ name: "Films", depth: 0 },
+                              { name: "Japanese films", depth: 1 },
+                              { name: "French films", depth: 1 })
+    end
+
+    it "collects section statistics" do
+      stats = @index.section_stats
+      expect(stats).to include(["Plot", 1], ["Synopsis", 1], ["Career", 1])
+    end
+
+    it "scopes section statistics to a category" do
+      stats = @index.section_stats(category: "Films", depth: 1)
+      headings = stats.map(&:first)
+      expect(headings).to include("Plot", "Synopsis")
+      expect(headings).not_to include("Career")
+    end
+
+    it "records redirect targets" do
+      db = SQLite3::Database.new(@db_path)
+      target = db.get_first_value("SELECT redirect_to FROM pages WHERE title = 'Old Film'")
+      db.close
+      expect(target).to eq("Film A")
+    end
+
+    it "validates against the source dump file" do
+      expect(@index.built?).to be true
+      expect(@index.valid_for?(@multistream_path)).to be true
+    end
+
+    it "records the wp2txt version it was built with" do
+      expect(@index.stats[:built_with]).to eq(Wp2txt::VERSION)
+    end
+
+    it "sees headings even when a trailing HTML comment follows the closing markers" do
+      expect(@index.find_articles(has_section: "Career", use_aliases: false)).to eq(["Person X"])
+    end
+
+    it "ignores commented-out category links" do
+      expect(@index.categories_of("Person X")).to eq(["Japanese actors"])
+    end
+
+    it "builds atomically: the existing index survives an unfinished rebuild" do
+      expect(@index.built?).to be true
+      rebuilding = Wp2txt::MetadataIndex.new(@db_path)
+      rebuilding.prepare_build!
+      rebuilding.insert_batch(pages: [[99, "Partial", 0, nil, 10]], categories: [], sections: [], hierarchy: [])
+      rebuilding.close # abandon before finalize
+
+      survivor = Wp2txt::MetadataIndex.new(@db_path)
+      expect(survivor.built?).to be true
+      expect(survivor.find_articles(title_match: "Film A")).to eq(["Film A"])
+      expect(survivor.find_articles(title_match: "Partial")).to be_empty
+      survivor.close
+      FileUtils.rm_f(Dir.glob("#{@db_path}.building*"))
+    end
+
+    it "detects a changed source dump file" do
+      File.binwrite(@multistream_path, File.binread(@multistream_path) + "x")
+      expect(@index.valid_for?(@multistream_path)).to be false
+    end
+
+    it "reports not built for a missing index file" do
+      missing = Wp2txt::MetadataIndex.new(File.join(@dir, "nope.sqlite3"))
+      expect(missing.built?).to be false
+    end
+  end
+
+  describe "CLI option validation" do
+    it "rejects --in-category without --find-articles" do
+      expect do
+        Wp2txt::CLI.parse_options(["-L", "ja", "--in-category", "Films"])
+      end.to raise_error(SystemExit)
+    end
+
+    it "rejects combining --build-index with --find-articles" do
+      expect do
+        Wp2txt::CLI.parse_options(["-L", "ja", "--build-index", "--find-articles"])
+      end.to raise_error(SystemExit)
+    end
+
+    it "accepts --find-articles with filters" do
+      opts = Wp2txt::CLI.parse_options(["-L", "ja", "--find-articles", "--in-category", "Films", "--has-section", "Plot"])
+      expect(opts[:find_articles]).to be true
+      expect(opts[:in_category]).to eq("Films")
+      expect(opts[:has_section]).to eq("Plot")
+    end
+  end
+end

data/spec/multi_dump_attach_spec.rb

@@ -0,0 +1,174 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+require "tmpdir"
+require_relative "support/multistream_fixture"
+require_relative "support/meta_db_fixture"
+require_relative "../lib/wp2txt/corpus"
+require_relative "../lib/wp2txt/langlinks_importer"
+
+# query_sql's multi-dump ATTACH (design doc §2) and the langlinks join demo (§3)
+RSpec.describe "multi-dump ATTACH" do
+  include MultistreamFixture
+  include MetaDbFixture
+
+  EN_PAGES = [
+    [101, "Film A", 0, nil, 100],
+    [102, "Film B", 0, nil, 100],
+    [103, "Person X (actor)", 0, nil, 100]
+  ].freeze
+
+  EN_SECTIONS = [
+    [101, "Plot", 2, 1],
+    [101, "Reception", 2, 2],
+    [101, "Production", 2, 3],
+    [102, "Synopsis", 2, 1]
+  ].freeze
+
+  around do |example|
+    Dir.mktmpdir do |dir|
+      @dir = dir
+      @multistream_path, @index_path = create_fixture(dir)
+
+      ms_index = Wp2txt::MultistreamIndex.new(@index_path, use_cache: false, show_progress: false)
+      @db_path = Wp2txt::MetadataIndex.path_for(@multistream_path, cache_dir: dir)
+      Wp2txt::MetadataIndexBuilder.new(
+        @multistream_path, ms_index.stream_offsets,
+        db_path: @db_path, num_processes: 0
+      ).build
+
+      # Same-date "en" dump (attachable), different-date "fr" dump (mismatch)
+      @en_meta = create_meta_db(dir, lang: "en", date: "20260101",
+                                     pages: EN_PAGES, sections: EN_SECTIONS)
+      @fr_meta = create_meta_db(dir, lang: "fr", date: "20260202",
+                                     pages: [[201, "Film A (fr)", 0, nil, 50]])
+
+      @corpus = Wp2txt::Corpus.for_input(@multistream_path, cache_dir: dir)
+      example.run
+      @corpus.close
+    end
+  end
+
+  describe "attach argument validation" do
+    it "rejects malformed language codes (paths, traversal, injection)" do
+      ["../x", "en/x", "en;DROP", "EN", "e", "x" * 20, ""].each do |bad|
+        expect { @corpus.query_sql("SELECT 1", attach: [bad]) }
+          .to raise_error(ArgumentError, /invalid language code/), "expected #{bad.inspect} to be rejected"
+      end
+    end
+
+    it "rejects languages with no installed index" do
+      expect { @corpus.query_sql("SELECT 1", attach: ["ko"]) }
+        .to raise_error(ArgumentError, /no installed index/)
+    end
+
+    it "rejects attaching the main database's own language" do
+      expect { @corpus.query_sql("SELECT 1", attach: ["test"]) }
+        .to raise_error(ArgumentError, /cannot attach 'test'/)
+    end
+  end
+
+  describe "read-only cross-dump queries" do
+    it "attaches another language's meta DB as {lang}_meta" do
+      result = @corpus.query_sql("SELECT title FROM en_meta.pages ORDER BY page_id", attach: ["en"])
+      expect(result[:rows].flatten).to eq(["Film A", "Film B", "Person X (actor)"])
+    end
+
+    it "attaches the FTS DB as {lang}_fts when built" do
+      create_fts_db(@en_meta)
+      result = @corpus.query_sql("SELECT name FROM en_fts.sqlite_master", attach: ["en"])
+      expect(result[:rows].flatten).to include("fts_map")
+      expect(result[:attached].first[:fts]).to be true
+    end
+
+    it "reports attached metadata (lang, dump_name, built_with, fts) in the response" do
+      result = @corpus.query_sql("SELECT 1", attach: ["en"])
+      expect(result[:attached]).to eq([
+        { lang: "en", dump_name: "enwiki-20260101", built_with: Wp2txt::VERSION, fts: false }
+      ])
+    end
+
+    it "flags dump_mismatch when only a different-date dump is installed" do
+      result = @corpus.query_sql("SELECT 1", attach: ["fr"])
+      entry = result[:attached].first
+      expect(entry[:dump_name]).to eq("frwiki-20260202")
+      expect(entry[:dump_mismatch]).to be true
+    end
+
+    it "makes attached databases read-only at the driver level" do
+      attachments = @corpus.send(:resolve_attachments, ["en"])
+      db = @corpus.send(:build_readonly_connection, attach_fts: false, attachments: attachments)
+      expect { db.execute("INSERT INTO en_meta.pages VALUES (999, 'x', 0, NULL, 0)") }
+        .to raise_error(SQLite3::Exception, /readonly/i)
+      db.close
+    end
+
+    it "maps hyphenated language codes to underscored aliases (zh-yue → zh_yue_meta)" do
+      create_meta_db(@dir, lang: "zh-yue", date: "20260101",
+                     pages: [[301, "Cantonese Page", 0, nil, 10]])
+      result = @corpus.query_sql("SELECT title FROM zh_yue_meta.pages", attach: ["zh-yue"])
+      expect(result[:rows].flatten).to eq(["Cantonese Page"])
+      expect(result[:attached].first[:lang]).to eq("zh-yue")
+    end
+  end
+
+  describe "user SQL ATTACH/DETACH stays forbidden (regression)" do
+    it "rejects ATTACH in user SQL even when the attach argument is used" do
+      expect { @corpus.query_sql("SELECT 1; ATTACH DATABASE 'x' AS y", attach: ["en"]) }
+        .to raise_error(ArgumentError, /forbidden/)
+    end
+
+    it "rejects DETACH in user SQL" do
+      expect { @corpus.query_sql("SELECT 1; DETACH DATABASE en_meta", attach: ["en"]) }
+        .to raise_error(ArgumentError, /forbidden/)
+    end
+
+    it "still allows 'attach' inside string literals" do
+      result = @corpus.query_sql("SELECT 'please attach the file' AS note")
+      expect(result[:rows].first.first).to eq("please attach the file")
+    end
+  end
+
+  describe "langlinks join demo (design doc §3)" do
+    before do
+      path = File.join(@dir, "testwiki-20260101-langlinks.sql")
+      File.write(path, <<~SQL)
+        INSERT INTO `langlinks` VALUES (1,'en','Film A'),(2,'en','Film B'),(3,'en','Person X (actor)');
+      SQL
+      Wp2txt::LanglinksImporter.new(@db_path, cache_dir: @dir).import!(path)
+    end
+
+    it "compares section structures of article pairs across languages in one query" do
+      result = @corpus.query_sql(<<~SQL, attach: ["en"])
+        SELECT p.title AS ja_title, ll.ll_title AS en_title,
+               (SELECT COUNT(*) FROM page_sections s WHERE s.page_id = p.page_id) AS ja_sections,
+               (SELECT COUNT(*) FROM en_meta.page_sections s2
+                JOIN en_meta.pages p2 ON p2.page_id = s2.page_id
+                WHERE p2.title = ll.ll_title) AS en_sections
+        FROM pages p
+        JOIN langlinks ll ON ll.ll_from = p.page_id AND ll.ll_lang = 'en'
+        WHERE p.namespace = 0 AND p.redirect_to IS NULL
+        LIMIT 20
+      SQL
+
+      expect(result[:columns]).to eq(%w[ja_title en_title ja_sections en_sections])
+      expect(result[:rows]).to contain_exactly(
+        ["Film A", "Film A", 2, 3],
+        ["Film B", "Film B", 1, 1],
+        ["Person X", "Person X (actor)", 2, 0]
+      )
+    end
+
+    it "exposes langlinks provenance via dump_info" do
+      info = @corpus.dump_info
+      expect(info[:langlinks][:source]).to eq("testwiki-20260101-langlinks.sql")
+      expect(info[:langlinks][:row_count]).to eq(3)
+    end
+
+    it "includes the langlinks table in describe_schema (introspection-based)" do
+      schemas = @corpus.describe_schema[:meta].join("\n")
+      expect(schemas).to include("CREATE TABLE langlinks")
+      expect(schemas).to include("idx_langlinks_from")
+    end
+  end
+end

data/spec/support/meta_db_fixture.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require "sqlite3"
+require_relative "../../lib/wp2txt/metadata_index"
+require_relative "../../lib/wp2txt/fts_index"
+require_relative "../../lib/wp2txt/version"
+
+# Synthetic Tier 1 / Tier 2 index DBs for cross-dump (ATTACH) and langlinks
+# specs: builds the SQLite files directly, without a real multistream dump
+module MetaDbFixture
+  # Create a synthetic built metadata DB for LANG in DIR, mimicking what
+  # MetadataIndex.path_for + MetadataIndexBuilder produce.
+  # @param pages [Array] rows of [page_id, title, namespace, redirect_to, text_length]
+  # @param sections [Array] rows of [page_id, heading, level, ord]
+  # @return [String] db path
+  def create_meta_db(dir, lang:, date:, pages: [], sections: [],
+                     built_with: Wp2txt::VERSION, built_at: "2026-01-02T00:00:00Z")
+    path = File.join(dir, "#{lang}wiki-#{date}-pages-articles-multistream_deadbeef#{Wp2txt::MetadataIndex::CACHE_SUFFIX}")
+    db = SQLite3::Database.new(path)
+    db.execute("CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT)")
+    db.execute("CREATE TABLE pages (page_id INTEGER PRIMARY KEY, title TEXT, namespace INTEGER, redirect_to TEXT, text_length INTEGER)")
+    db.execute("CREATE TABLE page_categories (page_id INTEGER, category TEXT)")
+    db.execute("CREATE TABLE page_sections (page_id INTEGER, heading TEXT, level INTEGER, ord INTEGER)")
+    db.execute("CREATE TABLE category_hierarchy (child TEXT, parent TEXT)")
+    pages.each { |r| db.execute("INSERT INTO pages VALUES (?, ?, ?, ?, ?)", r) }
+    sections.each { |r| db.execute("INSERT INTO page_sections VALUES (?, ?, ?, ?)", r) }
+    {
+      "schema_version" => Wp2txt::MetadataIndex::SCHEMA_VERSION.to_s,
+      "wp2txt_version" => built_with,
+      "dump_name" => "#{lang}wiki-#{date}",
+      "built_at" => built_at
+    }.each { |k, v| db.execute("INSERT INTO metadata VALUES (?, ?)", [k, v]) }
+    db.close
+    path
+  end
+
+  # Create the synthetic FTS DB companion of a meta DB created above
+  # (same basename, _fts suffix)
+  # @return [String] db path
+  def create_fts_db(meta_db_path, built_with: Wp2txt::VERSION, built_at: "2026-01-02T00:00:00Z")
+    path = meta_db_path.sub(/#{Wp2txt::MetadataIndex::CACHE_SUFFIX}\z/, Wp2txt::FtsIndex::CACHE_SUFFIX)
+    db = SQLite3::Database.new(path)
+    db.execute("CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT)")
+    db.execute("CREATE TABLE fts_map (rowid INTEGER PRIMARY KEY, page_id INTEGER, heading TEXT, ord INTEGER)")
+    {
+      "schema_version" => Wp2txt::FtsIndex::SCHEMA_VERSION.to_s,
+      "wp2txt_version" => built_with,
+      "built_at" => built_at
+    }.each { |k, v| db.execute("INSERT INTO metadata VALUES (?, ?)", [k, v]) }
+    db.close
+    path
+  end
+end

data/spec/support/multistream_fixture.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require "open3"
+
+# Synthetic two-stream multistream dump used by metadata index / corpus specs.
+# Stream 1 holds articles (ns 0), stream 2 holds category pages (ns 14).
+module MultistreamFixture
+  def page_xml(id:, ns:, title:, text:)
+    escaped = text.gsub("&", "&amp;").gsub("<", "&lt;").gsub(">", "&gt;")
+    <<~XML
+      <page>
+        <title>#{title}</title>
+        <ns>#{ns}</ns>
+        <id>#{id}</id>
+        <revision>
+          <id>#{id * 100}</id>
+          <text bytes="#{text.bytesize}">#{escaped}</text>
+        </revision>
+      </page>
+    XML
+  end
+
+  def bzip2(data)
+    out, status = Open3.capture2("bzip2", "-c", stdin_data: data)
+    raise "bzip2 failed" unless status.success?
+
+    out
+  end
+
+  # @return [Array(String, String)] [multistream_path, index_path]
+  def create_fixture(dir)
+    stream1_pages = [
+      page_xml(id: 1, ns: 0, title: "Film A",
+               text: "Intro.\n== Plot ==\nStory here.\n== Reception ==\nGood.\n[[Category:Japanese films]]\n"),
+      page_xml(id: 2, ns: 0, title: "Film B",
+               text: "Intro.\n== Synopsis ==\nStory here.\n[[Category:French films|B]]\n"),
+      page_xml(id: 3, ns: 0, title: "Person X",
+               text: "Bio.\n== Career == <!-- legacy anchor -->\nActing.\n== '''Style''' ==\nDistinct prose.\n<!-- [[Category:Hidden]] -->\n[[Category:Japanese actors]]\n"),
+      page_xml(id: 4, ns: 0, title: "Old Film",
+               text: "#REDIRECT [[Film A]]\n[[Category:Japanese films]]\n")
+    ]
+    stream2_pages = [
+      page_xml(id: 5, ns: 14, title: "Category:Japanese films", text: "[[Category:Films]]\n"),
+      page_xml(id: 6, ns: 14, title: "Category:French films", text: "[[Category:Films]]\n"),
+      page_xml(id: 7, ns: 14, title: "Category:Films", text: "Top category.\n"),
+      page_xml(id: 8, ns: 14, title: "Category:Japanese actors", text: "[[Category:People]]\n")
+    ]
+
+    stream1 = bzip2(stream1_pages.join)
+    stream2 = bzip2(stream2_pages.join)
+
+    multistream_path = File.join(dir, "testwiki-20260101-pages-articles-multistream.xml.bz2")
+    File.binwrite(multistream_path, stream1 + stream2)
+
+    offset2 = stream1.bytesize
+    index_lines = [
+      "0:1:Film A", "0:2:Film B", "0:3:Person X", "0:4:Old Film",
+      "#{offset2}:5:Category:Japanese films", "#{offset2}:6:Category:French films",
+      "#{offset2}:7:Category:Films", "#{offset2}:8:Category:Japanese actors"
+    ]
+    index_path = File.join(dir, "testwiki-20260101-pages-articles-multistream-index.txt")
+    File.write(index_path, index_lines.join("\n") + "\n")
+
+    [multistream_path, index_path]
+  end
+end