wp2txt 2.2.0 → 2.3.0

data/spec/langlinks_importer_spec.rb

@@ -0,0 +1,308 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+require "tmpdir"
+require "zlib"
+require_relative "support/multistream_fixture"
+require_relative "support/meta_db_fixture"
+require_relative "../lib/wp2txt/langlinks_importer"
+require_relative "../lib/wp2txt/multistream"
+
+RSpec.describe Wp2txt::LanglinksImporter do
+  include MultistreamFixture
+  include MetaDbFixture
+
+  around do |example|
+    Dir.mktmpdir do |dir|
+      @dir = dir
+      @multistream_path, @index_path = create_fixture(dir)
+
+      ms_index = Wp2txt::MultistreamIndex.new(@index_path, use_cache: false, show_progress: false)
+      @db_path = Wp2txt::MetadataIndex.path_for(@multistream_path, cache_dir: dir)
+      Wp2txt::MetadataIndexBuilder.new(
+        @multistream_path, ms_index.stream_offsets,
+        db_path: @db_path, num_processes: 0
+      ).build
+      example.run
+    end
+  end
+
+  # MySQL dump content covering the tricky cases: escaped quotes/backslashes,
+  # commas and parens inside titles, underscores, multiple INSERT statements,
+  # and an INSERT for a different table (must be ignored)
+  LANGLINKS_SQL = <<~SQL
+    -- MySQL dump fixture
+    CREATE TABLE `langlinks` (
+      `ll_from` int unsigned NOT NULL DEFAULT 0,
+      `ll_lang` varbinary(20) NOT NULL DEFAULT '',
+      `ll_title` varbinary(255) NOT NULL DEFAULT ''
+    ) ENGINE=InnoDB DEFAULT CHARSET=binary;
+
+    INSERT INTO `langlinks` VALUES (1,'en','Film A'),(1,'de','Film A (Film)'),(2,'en','It\\'s a Film, Really (1984)');
+    INSERT INTO `langlinks` VALUES (3,'en','Back\\\\slash Title'),(3,'fr','Film B'),(4,'en','Underscore_title');
+    INSERT INTO `pagelinks` VALUES (1,'Foo',0);
+    UNLOCK TABLES;
+  SQL
+
+  def write_langlinks(name, content = LANGLINKS_SQL, gzip: false)
+    path = File.join(@dir, name)
+    if gzip
+      Zlib::GzipWriter.open(path) { |gz| gz.write(content) }
+    else
+      File.write(path, content)
+    end
+    path
+  end
+
+  def import(path, **opts)
+    described_class.new(@db_path, cache_dir: @dir).import!(path, **opts)
+  end
+
+  def langlinks_rows
+    db = SQLite3::Database.new(@db_path, readonly: true)
+    rows = db.execute("SELECT ll_from, ll_lang, ll_title FROM langlinks ORDER BY ll_from, ll_lang")
+    db.close
+    rows
+  end
+
+  describe "parsing and normalization" do
+    it "imports tuples with escapes, commas, parens, and multiple INSERT statements" do
+      path = write_langlinks("testwiki-20260101-langlinks.sql")
+      result = import(path)
+
+      expect(result[:status]).to eq(:imported)
+      expect(result[:row_count]).to eq(6)
+      expect(langlinks_rows).to contain_exactly(
+        [1, "en", "Film A"],
+        [1, "de", "Film A (Film)"],
+        [2, "en", "It's a Film, Really (1984)"],
+        [3, "en", 'Back\slash Title'],
+        [3, "fr", "Film B"],
+        [4, "en", "Underscore title"]
+      )
+    end
+
+    it "reads .sql.gz files" do
+      path = write_langlinks("testwiki-20260101-langlinks.sql.gz", gzip: true)
+      expect(import(path)[:row_count]).to eq(6)
+    end
+
+    it "creates both indexes after the load" do
+      path = write_langlinks("testwiki-20260101-langlinks.sql")
+      import(path)
+      db = SQLite3::Database.new(@db_path, readonly: true)
+      indexes = db.execute("SELECT name FROM sqlite_master WHERE type = 'index' AND tbl_name = 'langlinks'").flatten
+      db.close
+      expect(indexes).to contain_exactly("idx_langlinks_from", "idx_langlinks_lang_title")
+    end
+
+    it "keeps all rows across batch boundaries" do
+      stub_const("Wp2txt::LanglinksImporter::BATCH_SIZE", 2)
+      path = write_langlinks("testwiki-20260101-langlinks.sql")
+      expect(import(path)[:row_count]).to eq(6)
+      expect(langlinks_rows.size).to eq(6)
+    end
+
+    it "imports multibyte (ja/zh/ko) titles, including escapes and underscores" do
+      sql = <<~SQL
+        INSERT INTO `langlinks` VALUES (1,'ja','宇宙戦艦ヤマト'),(2,'zh','粵語標題（電影）'),(3,'ko','한국어_제목'),(4,'ja','It\\'s 映画, Really（1984）');
+      SQL
+      path = write_langlinks("testwiki-20260101-langlinks.sql", sql)
+      result = import(path)
+
+      expect(result[:row_count]).to eq(4)
+      expect(langlinks_rows).to contain_exactly(
+        [1, "ja", "宇宙戦艦ヤマト"],
+        [2, "zh", "粵語標題（電影）"],
+        [3, "ko", "한국어 제목"],
+        [4, "ja", "It's 映画, Really（1984）"]
+      )
+    end
+
+    it "parses large multibyte INSERT lines fast enough (performance regression)" do
+      # A single ~200KB+ extended INSERT line full of multibyte titles: the
+      # old character-index parser was O(n²) here (minutes), regex scan is O(n)
+      titles = ["宇宙戦艦ヤマト（映画）", "粵語標題（電影, 1984）", "한국어 제목", "時間の旅, それから"]
+      tuples = Array.new(6_000) { |i| "(#{i + 1},'ja','#{titles[i % titles.size]}')" }
+      sql = +"INSERT INTO `langlinks` VALUES " << tuples.join(",") << ";\n"
+      expect(sql.bytesize).to be > 200_000
+
+      path = write_langlinks("testwiki-20260101-langlinks.sql", sql)
+      importer = described_class.new(@db_path, cache_dir: @dir)
+      count = 0
+      start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      importer.send(:each_source_row, path) { count += 1 }
+      elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start
+
+      expect(count).to eq(6_000)
+      expect(elapsed).to be < 0.5
+    end
+
+    it "skips and counts rows whose title contains invalid UTF-8 bytes" do
+      # Real dumps contain historically corrupted bytes in ll_title (VARBINARY)
+      sql = +"INSERT INTO `langlinks` VALUES (1,'en','Film A'),(2,'en','Bad\xFFTitle'),(3,'fr','Film B');\n"
+      path = write_langlinks("testwiki-20260101-langlinks.sql", sql.b)
+      result = import(path)
+
+      expect(result[:status]).to eq(:imported)
+      expect(result[:row_count]).to eq(2)
+      expect(result[:skipped_invalid]).to eq(1)
+      expect(langlinks_rows).to contain_exactly([1, "en", "Film A"], [3, "fr", "Film B"])
+      expect(result[:provenance][:skipped_invalid]).to eq(1)
+    end
+
+    it "skips and counts rows whose language code contains invalid UTF-8 bytes" do
+      sql = +"INSERT INTO `langlinks` VALUES (1,'e\xFFn','Film A'),(2,'en','Film B');\n"
+      path = write_langlinks("testwiki-20260101-langlinks.sql", sql.b)
+      result = import(path)
+
+      expect(result[:row_count]).to eq(1)
+      expect(result[:skipped_invalid]).to eq(1)
+      expect(langlinks_rows).to eq([[2, "en", "Film B"]])
+    end
+
+    it "handles invalid bytes in .sql.gz input as well" do
+      sql = +"INSERT INTO `langlinks` VALUES (1,'en','Bad\xFFTitle'),(2,'en','Film B');\n"
+      path = write_langlinks("testwiki-20260101-langlinks.sql.gz", sql.b, gzip: true)
+      result = import(path)
+
+      expect(result[:row_count]).to eq(1)
+      expect(result[:skipped_invalid]).to eq(1)
+    end
+
+    it "filters target languages with the langs option" do
+      path = write_langlinks("testwiki-20260101-langlinks.sql")
+      result = import(path, langs: %w[en fr])
+
+      expect(result[:row_count]).to eq(5)
+      expect(langlinks_rows.map { |r| r[1] }.uniq).to contain_exactly("en", "fr")
+      expect(result[:provenance][:lang_filter]).to eq("en,fr")
+    end
+  end
+
+  describe "version pinning" do
+    it "rejects a langlinks file whose dump name differs from the metadata DB" do
+      path = write_langlinks("testwiki-20260102-langlinks.sql")
+      expect { import(path) }.to raise_error(ArgumentError, /version mismatch/)
+    end
+
+    it "rejects a mismatched file even with force (no override)" do
+      path = write_langlinks("testwiki-20260102-langlinks.sql")
+      expect { import(path, force: true) }.to raise_error(ArgumentError, /version mismatch/)
+    end
+
+    it "rejects files whose name carries no dump name" do
+      path = write_langlinks("langlinks.sql")
+      expect { import(path) }.to raise_error(ArgumentError, /version mismatch/)
+    end
+  end
+
+  describe "re-import" do
+    it "is a no-op when already imported (reports imported_at)" do
+      path = write_langlinks("testwiki-20260101-langlinks.sql")
+      first = import(path)
+      second = import(path)
+
+      expect(second[:status]).to eq(:already_imported)
+      expect(second[:imported_at]).to eq(first[:provenance][:imported_at])
+      expect(second[:row_count]).to eq(6)
+    end
+
+    it "drops and re-imports with force, refreshing the provenance" do
+      path = write_langlinks("testwiki-20260101-langlinks.sql")
+      import(path)
+
+      smaller = <<~SQL
+        INSERT INTO `langlinks` VALUES (1,'en','Film A');
+      SQL
+      path2 = write_langlinks("testwiki-20260101-langlinks.sql", smaller)
+      result = import(path2, force: true)
+
+      expect(result[:status]).to eq(:imported)
+      expect(result[:row_count]).to eq(1)
+      expect(langlinks_rows).to eq([[1, "en", "Film A"]])
+      expect(result[:provenance][:row_count]).to eq(1)
+    end
+
+    it "clears stale provenance when a forced re-import fails midway" do
+      path = write_langlinks("testwiki-20260101-langlinks.sql")
+      expect(import(path)[:status]).to eq(:imported)
+
+      # Simulate a failure during the load (corrupt bytes, disk full, ...),
+      # after the first batch has already been committed
+      stub_const("Wp2txt::LanglinksImporter::BATCH_SIZE", 1)
+      failing = described_class.new(@db_path, cache_dir: @dir)
+      allow(failing).to receive(:each_source_row) do |_source, &block|
+        block.call(1, "en", "Film A")
+        raise IOError, "simulated read failure"
+      end
+      expect { failing.import!(path, force: true) }.to raise_error(IOError)
+
+      # Partial table, but provenance is gone: judged as "not imported"
+      expect(langlinks_rows).to eq([[1, "en", "Film A"]])
+      meta = Wp2txt::MetadataIndex.new(@db_path)
+      expect(meta.langlinks_provenance).to be_nil
+      meta.close
+
+      # A non-force retry re-imports instead of reporting a stale success
+      retry_result = import(path)
+      expect(retry_result[:status]).to eq(:imported)
+      expect(retry_result[:row_count]).to eq(6)
+    end
+  end
+
+  describe "provenance" do
+    it "stamps source, size, time, tool version, filter, and row count" do
+      path = write_langlinks("testwiki-20260101-langlinks.sql")
+      prov = import(path)[:provenance]
+
+      expect(prov[:source]).to eq("testwiki-20260101-langlinks.sql")
+      expect(prov[:source_size]).to eq(File.size(path))
+      expect(prov[:imported_at]).to match(/\A\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}/)
+      expect(prov[:imported_with]).to eq(Wp2txt::VERSION)
+      expect(prov[:lang_filter]).to eq("all")
+      expect(prov[:row_count]).to eq(6)
+
+      meta = Wp2txt::MetadataIndex.new(@db_path)
+      expect(meta.langlinks_provenance[:row_count]).to eq(6)
+      meta.close
+    end
+  end
+
+  describe "sanity check" do
+    it "is skipped when the target language has no local meta DB" do
+      path = write_langlinks("testwiki-20260101-langlinks.sql")
+      expect(import(path)[:sanity]).to eq([])
+    end
+
+    it "reports the join rate against an installed target meta DB" do
+      create_meta_db(@dir, lang: "en", date: "20260101",
+                     pages: [[101, "Film A", 0, nil, 10], [102, "Film B", 0, nil, 10]])
+      sql = <<~SQL
+        INSERT INTO `langlinks` VALUES (1,'en','Film A'),(2,'en','Film B');
+      SQL
+      path = write_langlinks("testwiki-20260101-langlinks.sql", sql)
+      result = import(path)
+
+      expect(result[:sanity].size).to eq(1)
+      check = result[:sanity].first
+      expect(check[:lang]).to eq("en")
+      expect(check[:match_rate]).to eq(1.0)
+      expect(check[:warning]).to be false
+      expect(check[:against]).to eq("enwiki-20260101")
+    end
+
+    it "warns when the join rate falls below 90%" do
+      create_meta_db(@dir, lang: "en", date: "20260101",
+                     pages: [[101, "Something Else", 0, nil, 10]])
+      sql = <<~SQL
+        INSERT INTO `langlinks` VALUES (1,'en','Film A'),(2,'en','Film B');
+      SQL
+      path = write_langlinks("testwiki-20260101-langlinks.sql", sql)
+      check = import(path)[:sanity].first
+
+      expect(check[:match_rate]).to eq(0.0)
+      expect(check[:warning]).to be true
+    end
+  end
+end

data/spec/multi_dump_attach_spec.rb

@@ -0,0 +1,174 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+require "tmpdir"
+require_relative "support/multistream_fixture"
+require_relative "support/meta_db_fixture"
+require_relative "../lib/wp2txt/corpus"
+require_relative "../lib/wp2txt/langlinks_importer"
+
+# query_sql's multi-dump ATTACH (design doc §2) and the langlinks join demo (§3)
+RSpec.describe "multi-dump ATTACH" do
+  include MultistreamFixture
+  include MetaDbFixture
+
+  EN_PAGES = [
+    [101, "Film A", 0, nil, 100],
+    [102, "Film B", 0, nil, 100],
+    [103, "Person X (actor)", 0, nil, 100]
+  ].freeze
+
+  EN_SECTIONS = [
+    [101, "Plot", 2, 1],
+    [101, "Reception", 2, 2],
+    [101, "Production", 2, 3],
+    [102, "Synopsis", 2, 1]
+  ].freeze
+
+  around do |example|
+    Dir.mktmpdir do |dir|
+      @dir = dir
+      @multistream_path, @index_path = create_fixture(dir)
+
+      ms_index = Wp2txt::MultistreamIndex.new(@index_path, use_cache: false, show_progress: false)
+      @db_path = Wp2txt::MetadataIndex.path_for(@multistream_path, cache_dir: dir)
+      Wp2txt::MetadataIndexBuilder.new(
+        @multistream_path, ms_index.stream_offsets,
+        db_path: @db_path, num_processes: 0
+      ).build
+
+      # Same-date "en" dump (attachable), different-date "fr" dump (mismatch)
+      @en_meta = create_meta_db(dir, lang: "en", date: "20260101",
+                                     pages: EN_PAGES, sections: EN_SECTIONS)
+      @fr_meta = create_meta_db(dir, lang: "fr", date: "20260202",
+                                     pages: [[201, "Film A (fr)", 0, nil, 50]])
+
+      @corpus = Wp2txt::Corpus.for_input(@multistream_path, cache_dir: dir)
+      example.run
+      @corpus.close
+    end
+  end
+
+  describe "attach argument validation" do
+    it "rejects malformed language codes (paths, traversal, injection)" do
+      ["../x", "en/x", "en;DROP", "EN", "e", "x" * 20, ""].each do |bad|
+        expect { @corpus.query_sql("SELECT 1", attach: [bad]) }
+          .to raise_error(ArgumentError, /invalid language code/), "expected #{bad.inspect} to be rejected"
+      end
+    end
+
+    it "rejects languages with no installed index" do
+      expect { @corpus.query_sql("SELECT 1", attach: ["ko"]) }
+        .to raise_error(ArgumentError, /no installed index/)
+    end
+
+    it "rejects attaching the main database's own language" do
+      expect { @corpus.query_sql("SELECT 1", attach: ["test"]) }
+        .to raise_error(ArgumentError, /cannot attach 'test'/)
+    end
+  end
+
+  describe "read-only cross-dump queries" do
+    it "attaches another language's meta DB as {lang}_meta" do
+      result = @corpus.query_sql("SELECT title FROM en_meta.pages ORDER BY page_id", attach: ["en"])
+      expect(result[:rows].flatten).to eq(["Film A", "Film B", "Person X (actor)"])
+    end
+
+    it "attaches the FTS DB as {lang}_fts when built" do
+      create_fts_db(@en_meta)
+      result = @corpus.query_sql("SELECT name FROM en_fts.sqlite_master", attach: ["en"])
+      expect(result[:rows].flatten).to include("fts_map")
+      expect(result[:attached].first[:fts]).to be true
+    end
+
+    it "reports attached metadata (lang, dump_name, built_with, fts) in the response" do
+      result = @corpus.query_sql("SELECT 1", attach: ["en"])
+      expect(result[:attached]).to eq([
+        { lang: "en", dump_name: "enwiki-20260101", built_with: Wp2txt::VERSION, fts: false }
+      ])
+    end
+
+    it "flags dump_mismatch when only a different-date dump is installed" do
+      result = @corpus.query_sql("SELECT 1", attach: ["fr"])
+      entry = result[:attached].first
+      expect(entry[:dump_name]).to eq("frwiki-20260202")
+      expect(entry[:dump_mismatch]).to be true
+    end
+
+    it "makes attached databases read-only at the driver level" do
+      attachments = @corpus.send(:resolve_attachments, ["en"])
+      db = @corpus.send(:build_readonly_connection, attach_fts: false, attachments: attachments)
+      expect { db.execute("INSERT INTO en_meta.pages VALUES (999, 'x', 0, NULL, 0)") }
+        .to raise_error(SQLite3::Exception, /readonly/i)
+      db.close
+    end
+
+    it "maps hyphenated language codes to underscored aliases (zh-yue → zh_yue_meta)" do
+      create_meta_db(@dir, lang: "zh-yue", date: "20260101",
+                     pages: [[301, "Cantonese Page", 0, nil, 10]])
+      result = @corpus.query_sql("SELECT title FROM zh_yue_meta.pages", attach: ["zh-yue"])
+      expect(result[:rows].flatten).to eq(["Cantonese Page"])
+      expect(result[:attached].first[:lang]).to eq("zh-yue")
+    end
+  end
+
+  describe "user SQL ATTACH/DETACH stays forbidden (regression)" do
+    it "rejects ATTACH in user SQL even when the attach argument is used" do
+      expect { @corpus.query_sql("SELECT 1; ATTACH DATABASE 'x' AS y", attach: ["en"]) }
+        .to raise_error(ArgumentError, /forbidden/)
+    end
+
+    it "rejects DETACH in user SQL" do
+      expect { @corpus.query_sql("SELECT 1; DETACH DATABASE en_meta", attach: ["en"]) }
+        .to raise_error(ArgumentError, /forbidden/)
+    end
+
+    it "still allows 'attach' inside string literals" do
+      result = @corpus.query_sql("SELECT 'please attach the file' AS note")
+      expect(result[:rows].first.first).to eq("please attach the file")
+    end
+  end
+
+  describe "langlinks join demo (design doc §3)" do
+    before do
+      path = File.join(@dir, "testwiki-20260101-langlinks.sql")
+      File.write(path, <<~SQL)
+        INSERT INTO `langlinks` VALUES (1,'en','Film A'),(2,'en','Film B'),(3,'en','Person X (actor)');
+      SQL
+      Wp2txt::LanglinksImporter.new(@db_path, cache_dir: @dir).import!(path)
+    end
+
+    it "compares section structures of article pairs across languages in one query" do
+      result = @corpus.query_sql(<<~SQL, attach: ["en"])
+        SELECT p.title AS ja_title, ll.ll_title AS en_title,
+               (SELECT COUNT(*) FROM page_sections s WHERE s.page_id = p.page_id) AS ja_sections,
+               (SELECT COUNT(*) FROM en_meta.page_sections s2
+                JOIN en_meta.pages p2 ON p2.page_id = s2.page_id
+                WHERE p2.title = ll.ll_title) AS en_sections
+        FROM pages p
+        JOIN langlinks ll ON ll.ll_from = p.page_id AND ll.ll_lang = 'en'
+        WHERE p.namespace = 0 AND p.redirect_to IS NULL
+        LIMIT 20
+      SQL
+
+      expect(result[:columns]).to eq(%w[ja_title en_title ja_sections en_sections])
+      expect(result[:rows]).to contain_exactly(
+        ["Film A", "Film A", 2, 3],
+        ["Film B", "Film B", 1, 1],
+        ["Person X", "Person X (actor)", 2, 0]
+      )
+    end
+
+    it "exposes langlinks provenance via dump_info" do
+      info = @corpus.dump_info
+      expect(info[:langlinks][:source]).to eq("testwiki-20260101-langlinks.sql")
+      expect(info[:langlinks][:row_count]).to eq(3)
+    end
+
+    it "includes the langlinks table in describe_schema (introspection-based)" do
+      schemas = @corpus.describe_schema[:meta].join("\n")
+      expect(schemas).to include("CREATE TABLE langlinks")
+      expect(schemas).to include("idx_langlinks_from")
+    end
+  end
+end

data/spec/support/meta_db_fixture.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require "sqlite3"
+require_relative "../../lib/wp2txt/metadata_index"
+require_relative "../../lib/wp2txt/fts_index"
+require_relative "../../lib/wp2txt/version"
+
+# Synthetic Tier 1 / Tier 2 index DBs for cross-dump (ATTACH) and langlinks
+# specs: builds the SQLite files directly, without a real multistream dump
+module MetaDbFixture
+  # Create a synthetic built metadata DB for LANG in DIR, mimicking what
+  # MetadataIndex.path_for + MetadataIndexBuilder produce.
+  # @param pages [Array] rows of [page_id, title, namespace, redirect_to, text_length]
+  # @param sections [Array] rows of [page_id, heading, level, ord]
+  # @return [String] db path
+  def create_meta_db(dir, lang:, date:, pages: [], sections: [],
+                     built_with: Wp2txt::VERSION, built_at: "2026-01-02T00:00:00Z")
+    path = File.join(dir, "#{lang}wiki-#{date}-pages-articles-multistream_deadbeef#{Wp2txt::MetadataIndex::CACHE_SUFFIX}")
+    db = SQLite3::Database.new(path)
+    db.execute("CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT)")
+    db.execute("CREATE TABLE pages (page_id INTEGER PRIMARY KEY, title TEXT, namespace INTEGER, redirect_to TEXT, text_length INTEGER)")
+    db.execute("CREATE TABLE page_categories (page_id INTEGER, category TEXT)")
+    db.execute("CREATE TABLE page_sections (page_id INTEGER, heading TEXT, level INTEGER, ord INTEGER)")
+    db.execute("CREATE TABLE category_hierarchy (child TEXT, parent TEXT)")
+    pages.each { |r| db.execute("INSERT INTO pages VALUES (?, ?, ?, ?, ?)", r) }
+    sections.each { |r| db.execute("INSERT INTO page_sections VALUES (?, ?, ?, ?)", r) }
+    {
+      "schema_version" => Wp2txt::MetadataIndex::SCHEMA_VERSION.to_s,
+      "wp2txt_version" => built_with,
+      "dump_name" => "#{lang}wiki-#{date}",
+      "built_at" => built_at
+    }.each { |k, v| db.execute("INSERT INTO metadata VALUES (?, ?)", [k, v]) }
+    db.close
+    path
+  end
+
+  # Create the synthetic FTS DB companion of a meta DB created above
+  # (same basename, _fts suffix)
+  # @return [String] db path
+  def create_fts_db(meta_db_path, built_with: Wp2txt::VERSION, built_at: "2026-01-02T00:00:00Z")
+    path = meta_db_path.sub(/#{Wp2txt::MetadataIndex::CACHE_SUFFIX}\z/, Wp2txt::FtsIndex::CACHE_SUFFIX)
+    db = SQLite3::Database.new(path)
+    db.execute("CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT)")
+    db.execute("CREATE TABLE fts_map (rowid INTEGER PRIMARY KEY, page_id INTEGER, heading TEXT, ord INTEGER)")
+    {
+      "schema_version" => Wp2txt::FtsIndex::SCHEMA_VERSION.to_s,
+      "wp2txt_version" => built_with,
+      "built_at" => built_at
+    }.each { |k, v| db.execute("INSERT INTO metadata VALUES (?, ?)", [k, v]) }
+    db.close
+    path
+  end
+end