wp2txt 2.1.1 → 2.2.0

data/spec/fts_index_spec.rb

@@ -0,0 +1,245 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+require "tmpdir"
+require "fileutils"
+require "json"
+require_relative "support/multistream_fixture"
+require_relative "../lib/wp2txt/fts_index"
+require_relative "../lib/wp2txt/corpus"
+
+RSpec.describe "Wp2txt Full-Text Search" do
+  include MultistreamFixture
+
+  def build_indexes(dir, tokenizer: "unicode61", optimize: true)
+    multistream_path, index_path = create_fixture(dir)
+    ms_index = Wp2txt::MultistreamIndex.new(index_path, use_cache: false, show_progress: false)
+
+    meta_db = Wp2txt::MetadataIndex.path_for(multistream_path, cache_dir: dir)
+    Wp2txt::MetadataIndexBuilder.new(
+      multistream_path, ms_index.stream_offsets, db_path: meta_db, num_processes: 0
+    ).build
+
+    fts_db = Wp2txt::FtsIndex.path_for(multistream_path, cache_dir: dir)
+    Wp2txt::FtsIndexBuilder.new(
+      multistream_path, ms_index.stream_offsets,
+      db_path: fts_db, meta_db_path: meta_db, tokenizer: tokenizer,
+      num_processes: 0, optimize: optimize
+    ).build
+
+    [multistream_path, fts_db, meta_db]
+  end
+
+  describe Wp2txt::FtsIndex do
+    describe ".default_tokenizer" do
+      it "picks trigram for CJK language dumps" do
+        expect(described_class.default_tokenizer("/x/jawiki-20260701-multistream.xml.bz2")).to eq("trigram")
+        expect(described_class.default_tokenizer("/x/zhwiki-20260701-multistream.xml.bz2")).to eq("trigram")
+      end
+
+      it "picks unicode61 for space-delimited language dumps" do
+        expect(described_class.default_tokenizer("/x/enwiki-20260701-multistream.xml.bz2")).to eq("unicode61")
+        expect(described_class.default_tokenizer("/x/dewiki-20260701-multistream.xml.bz2")).to eq("unicode61")
+      end
+    end
+
+    context "with a built unicode61 index" do
+      around do |example|
+        Dir.mktmpdir do |dir|
+          @multistream_path, fts_db, meta_db = build_indexes(dir)
+          @fts = described_class.new(fts_db, meta_db)
+          example.run
+          @fts.close
+        end
+      end
+
+      it "is built, valid, and records the tokenizer" do
+        expect(@fts.built?).to be true
+        expect(@fts.valid_for?(@multistream_path)).to be true
+        expect(@fts.tokenizer).to eq("unicode61")
+        expect(@fts.stats[:section_count]).to be > 0
+      end
+
+      it "finds matches across articles with exact counts" do
+        result = @fts.search("Story", count: "exact")
+        expect(result[:total]).to eq(2)
+        expect(result[:total_is_capped]).to be false
+        expect(result[:hits].map { |h| h[:title] }).to contain_exactly("Film A", "Film B")
+      end
+
+      it "searches lead sections (empty heading)" do
+        result = @fts.search("Intro", count: "exact")
+        expect(result[:total]).to eq(2)
+        expect(result[:hits].map { |h| h[:heading] }.uniq).to eq([""])
+      end
+
+      it "excludes redirect pages from the index" do
+        result = @fts.search("Old Film", count: "exact")
+        expect(result[:total]).to eq(0)
+      end
+
+      it "returns zero for absent strings (absence claim)" do
+        expect(@fts.search("zebra unicorn", count: "exact")[:total]).to eq(0)
+      end
+
+      it "composes with an exact category filter" do
+        result = @fts.search("Story", category: "Japanese films", count: "exact")
+        expect(result[:hits].map { |h| h[:title] }).to eq(["Film A"])
+      end
+
+      it "composes with a recursive category filter" do
+        result = @fts.search("Story", category: "Films", depth: 1, count: "exact")
+        expect(result[:total]).to eq(2)
+      end
+
+      it "composes with a section filter" do
+        result = @fts.search("Story", sections: ["Plot"], count: "exact")
+        expect(result[:hits].map { |h| h[:heading] }).to eq(["Plot"])
+      end
+
+      it "normalizes decorated headings the same way as the metadata index" do
+        result = @fts.search("Distinct prose", count: "exact")
+        expect(result[:hits].map { |h| h[:heading] }).to eq(["Style"])
+        expect(@fts.search("Distinct prose", sections: ["Style"], count: "exact")[:total]).to eq(1)
+      end
+
+      it "keeps ord aligned between page_sections and fts_map (shared semantics)" do
+        meta_db = SQLite3::Database.new(@fts.meta_db_path, readonly: true)
+        meta_ord = meta_db.get_first_value(
+          "SELECT ps.ord FROM page_sections ps JOIN pages p ON p.page_id = ps.page_id " \
+          "WHERE p.title = 'Person X' AND ps.heading = 'Career'"
+        )
+        meta_db.close
+        fts_db = SQLite3::Database.new(@fts.db_path, readonly: true)
+        fts_ord = fts_db.get_first_value(
+          "SELECT fm.ord FROM fts_map fm WHERE fm.heading = 'Career'"
+        )
+        fts_db.close
+        expect(meta_ord).to eq(1) # lead = 0 (not stored), first heading = 1
+        expect(fts_ord).to eq(meta_ord)
+      end
+
+      it "caps counting when requested" do
+        result = @fts.search("Story", count: "capped", count_cap: 1)
+        expect(result[:total]).to eq(1)
+        expect(result[:total_is_capped]).to be true
+      end
+
+      it "supports raw FTS5 query mode" do
+        result = @fts.search("Story OR Acting", mode: "query", count: "exact")
+        expect(result[:total]).to eq(3)
+      end
+
+      it "escapes quotes in phrase mode" do
+        expect { @fts.search('say "hi" now', count: "exact") }.not_to raise_error
+      end
+    end
+
+    context "built with optimize: false" do
+      around do |example|
+        Dir.mktmpdir do |dir|
+          @multistream_path, fts_db, meta_db = build_indexes(dir, optimize: false)
+          @fts = described_class.new(fts_db, meta_db)
+          example.run
+          @fts.close
+        end
+      end
+
+      it "is valid, flagged unoptimized, and fully searchable" do
+        expect(@fts.built?).to be true
+        expect(@fts.valid_for?(@multistream_path)).to be true
+        expect(@fts.optimized?).to be false
+        expect(@fts.stats[:optimized]).to be false
+        expect(@fts.search("Story", count: "exact")[:total]).to eq(2)
+      end
+
+      it "can be optimized afterwards (idempotent)" do
+        expect(@fts.optimize!).to be true
+        expect(@fts.optimized?).to be true
+        expect(@fts.search("Story", count: "exact")[:total]).to eq(2)
+        expect(@fts.optimize!).to be true
+      end
+    end
+
+    context "with a trigram index" do
+      around do |example|
+        Dir.mktmpdir do |dir|
+          @multistream_path, fts_db, meta_db = build_indexes(dir, tokenizer: "trigram")
+          @fts = described_class.new(fts_db, meta_db)
+          example.run
+          @fts.close
+        end
+      end
+
+      it "matches substrings of three or more characters" do
+        result = @fts.search("tory", count: "exact")
+        expect(result[:total]).to eq(2)
+      end
+    end
+  end
+
+  describe "Corpus#search_text" do
+    around do |example|
+      Dir.mktmpdir do |dir|
+        @multistream_path, = build_indexes(dir)
+        @corpus = Wp2txt::Corpus.for_input(@multistream_path, cache_dir: dir)
+        example.run
+        @corpus.close
+      end
+    end
+
+    it "reports the fulltext tier in dump_info" do
+      info = @corpus.dump_info
+      expect(info[:tiers][:fulltext]).to be true
+      expect(info[:fulltext_current]).to be true
+      expect(info[:fulltext][:tokenizer]).to eq("unicode61")
+    end
+
+    it "exposes the fts tables to query_sql" do
+      result = @corpus.query_sql("SELECT COUNT(*) FROM fts.fts_map")
+      expect(result[:rows].first.first).to be > 0
+      expect(@corpus.describe_schema[:fts].join).to include("fts_map")
+    end
+
+    it "returns hits with section paths and dump identity" do
+      result = @corpus.search_text("Story", count: "exact")
+      expect(result[:dump]).to eq("testwiki-20260101")
+      expect(result[:total]).to eq(2)
+      paths = result[:hits].map { |h| h[:section_path] }
+      expect(paths).to contain_exactly("Film A > Plot", "Film B > Synopsis")
+    end
+
+    it "renders snippets containing the search term" do
+      result = @corpus.search_text("Story")
+      expect(result[:hits].first[:snippet]).to include("Story")
+    end
+
+    it "uses the article title as section_path for lead hits" do
+      result = @corpus.search_text("Intro", count: "exact")
+      expect(result[:hits].map { |h| h[:section_path] }).to contain_exactly("Film A", "Film B")
+      expect(result[:hits].first).not_to have_key(:section)
+    end
+
+    it "expands section filters through alias sets" do
+      @corpus.save_alias_set("plot", [%w[Plot Synopsis]], min_articles: 1)
+      result = @corpus.search_text("Story", sections: ["Plot"], alias_set: "plot", count: "exact")
+      expect(result[:total]).to eq(2)
+    end
+
+    it "raises a helpful error when the index is missing" do
+      Dir.mktmpdir do |dir2|
+        ms2, = create_fixture(dir2)
+        ms_index = Wp2txt::MultistreamIndex.new(
+          ms2.sub(/multistream\.xml\.bz2\z/, "multistream-index.txt"), use_cache: false, show_progress: false
+        )
+        Wp2txt::MetadataIndexBuilder.new(
+          ms2, ms_index.stream_offsets,
+          db_path: Wp2txt::MetadataIndex.path_for(ms2, cache_dir: dir2), num_processes: 0
+        ).build
+        corpus2 = Wp2txt::Corpus.for_input(ms2, cache_dir: dir2)
+        expect { corpus2.search_text("x") }.to raise_error(ArgumentError, /Full-text index not built/)
+        corpus2.close
+      end
+    end
+  end
+end

data/spec/metadata_index_spec.rb

@@ -0,0 +1,208 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+require "tmpdir"
+require "fileutils"
+require "open3"
+require_relative "support/multistream_fixture"
+require_relative "../lib/wp2txt/metadata_index"
+require_relative "../lib/wp2txt/multistream"
+require_relative "../lib/wp2txt/cli"
+
+RSpec.describe "Wp2txt Metadata Index" do
+  include MultistreamFixture
+
+  describe Wp2txt::MetadataIndex do
+    describe ".normalize_category" do
+      it "replaces underscores, trims, and capitalizes the first letter" do
+        expect(described_class.normalize_category(" japanese_films ")).to eq("Japanese films")
+      end
+
+      it "leaves non-ASCII names unchanged" do
+        expect(described_class.normalize_category("日本の映画作品")).to eq("日本の映画作品")
+      end
+    end
+
+    describe ".clean_heading" do
+      it "strips bold markup and resolves links" do
+        expect(described_class.clean_heading("'''Bold''' [[link|Label]]")).to eq("Bold Label")
+      end
+
+      it "strips HTML tags" do
+        expect(described_class.clean_heading("<small>Notes</small>")).to eq("Notes")
+      end
+    end
+
+    describe ".expand_section_names" do
+      it "expands a canonical name to its alias group" do
+        expect(described_class.expand_section_names("Plot")).to include("Plot", "Synopsis")
+      end
+
+      it "expands an alias back to the full group (bidirectional)" do
+        expect(described_class.expand_section_names("Synopsis")).to include("Plot", "Synopsis")
+      end
+
+      it "returns the name itself when no alias group matches" do
+        expect(described_class.expand_section_names("Nonexistent Section")).to eq(["Nonexistent Section"])
+      end
+    end
+
+    describe ".path_for" do
+      it "builds a cache path keyed to the dump file" do
+        path = described_class.path_for("/dumps/jawiki-20260101-pages-articles-multistream.xml.bz2", cache_dir: "/cache")
+        expect(path).to start_with("/cache/jawiki-20260101-pages-articles-multistream")
+        expect(path).to end_with("_meta.sqlite3")
+      end
+    end
+  end
+
+  describe "build and query" do
+    around do |example|
+      Dir.mktmpdir do |dir|
+        @dir = dir
+        @multistream_path, @index_path = create_fixture(dir)
+        @db_path = File.join(dir, "meta.sqlite3")
+        ms_index = Wp2txt::MultistreamIndex.new(@index_path, use_cache: false, show_progress: false)
+        builder = Wp2txt::MetadataIndexBuilder.new(
+          @multistream_path, ms_index.stream_offsets,
+          db_path: @db_path, num_processes: 0
+        )
+        @index = builder.build
+        example.run
+        @index.close
+      end
+    end
+
+    it "records all pages and counts articles excluding redirects and category pages" do
+      stats = @index.stats
+      expect(stats[:page_count]).to eq(8)
+      expect(stats[:article_count]).to eq(3)
+      expect(stats[:dump_name]).to eq("testwiki-20260101")
+    end
+
+    it "finds articles by exact category" do
+      expect(@index.find_articles(category: "Japanese films")).to eq(["Film A"])
+    end
+
+    it "finds articles through subcategories with depth" do
+      expect(@index.find_articles(category: "Films", depth: 1)).to contain_exactly("Film A", "Film B")
+    end
+
+    it "does not include subcategory members at depth 0" do
+      expect(@index.find_articles(category: "Films", depth: 0)).to be_empty
+    end
+
+    it "matches sections through aliases" do
+      titles = @index.find_articles(category: "Films", depth: 1, has_section: "Plot")
+      expect(titles).to contain_exactly("Film A", "Film B")
+    end
+
+    it "matches sections exactly when aliases are disabled" do
+      titles = @index.find_articles(category: "Films", depth: 1, has_section: "Plot", use_aliases: false)
+      expect(titles).to eq(["Film A"])
+    end
+
+    it "excludes redirects from results" do
+      expect(@index.find_articles(title_match: "Old Film")).to be_empty
+    end
+
+    it "filters by title substring" do
+      expect(@index.find_articles(title_match: "Film")).to contain_exactly("Film A", "Film B")
+    end
+
+    it "applies limit and reports full count separately" do
+      titles = @index.find_articles(title_match: "Film", limit: 1)
+      expect(titles.size).to eq(1)
+      expect(@index.count_articles(title_match: "Film")).to eq(2)
+    end
+
+    it "returns the category tree with depths" do
+      tree = @index.category_tree("Films", depth: 1)
+      expect(tree).to include({ name: "Films", depth: 0 },
+                              { name: "Japanese films", depth: 1 },
+                              { name: "French films", depth: 1 })
+    end
+
+    it "collects section statistics" do
+      stats = @index.section_stats
+      expect(stats).to include(["Plot", 1], ["Synopsis", 1], ["Career", 1])
+    end
+
+    it "scopes section statistics to a category" do
+      stats = @index.section_stats(category: "Films", depth: 1)
+      headings = stats.map(&:first)
+      expect(headings).to include("Plot", "Synopsis")
+      expect(headings).not_to include("Career")
+    end
+
+    it "records redirect targets" do
+      db = SQLite3::Database.new(@db_path)
+      target = db.get_first_value("SELECT redirect_to FROM pages WHERE title = 'Old Film'")
+      db.close
+      expect(target).to eq("Film A")
+    end
+
+    it "validates against the source dump file" do
+      expect(@index.built?).to be true
+      expect(@index.valid_for?(@multistream_path)).to be true
+    end
+
+    it "records the wp2txt version it was built with" do
+      expect(@index.stats[:built_with]).to eq(Wp2txt::VERSION)
+    end
+
+    it "sees headings even when a trailing HTML comment follows the closing markers" do
+      expect(@index.find_articles(has_section: "Career", use_aliases: false)).to eq(["Person X"])
+    end
+
+    it "ignores commented-out category links" do
+      expect(@index.categories_of("Person X")).to eq(["Japanese actors"])
+    end
+
+    it "builds atomically: the existing index survives an unfinished rebuild" do
+      expect(@index.built?).to be true
+      rebuilding = Wp2txt::MetadataIndex.new(@db_path)
+      rebuilding.prepare_build!
+      rebuilding.insert_batch(pages: [[99, "Partial", 0, nil, 10]], categories: [], sections: [], hierarchy: [])
+      rebuilding.close # abandon before finalize
+
+      survivor = Wp2txt::MetadataIndex.new(@db_path)
+      expect(survivor.built?).to be true
+      expect(survivor.find_articles(title_match: "Film A")).to eq(["Film A"])
+      expect(survivor.find_articles(title_match: "Partial")).to be_empty
+      survivor.close
+      FileUtils.rm_f(Dir.glob("#{@db_path}.building*"))
+    end
+
+    it "detects a changed source dump file" do
+      File.binwrite(@multistream_path, File.binread(@multistream_path) + "x")
+      expect(@index.valid_for?(@multistream_path)).to be false
+    end
+
+    it "reports not built for a missing index file" do
+      missing = Wp2txt::MetadataIndex.new(File.join(@dir, "nope.sqlite3"))
+      expect(missing.built?).to be false
+    end
+  end
+
+  describe "CLI option validation" do
+    it "rejects --in-category without --find-articles" do
+      expect do
+        Wp2txt::CLI.parse_options(["-L", "ja", "--in-category", "Films"])
+      end.to raise_error(SystemExit)
+    end
+
+    it "rejects combining --build-index with --find-articles" do
+      expect do
+        Wp2txt::CLI.parse_options(["-L", "ja", "--build-index", "--find-articles"])
+      end.to raise_error(SystemExit)
+    end
+
+    it "accepts --find-articles with filters" do
+      opts = Wp2txt::CLI.parse_options(["-L", "ja", "--find-articles", "--in-category", "Films", "--has-section", "Plot"])
+      expect(opts[:find_articles]).to be true
+      expect(opts[:in_category]).to eq("Films")
+      expect(opts[:has_section]).to eq("Plot")
+    end
+  end
+end

data/spec/support/multistream_fixture.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require "open3"
+
+# Synthetic two-stream multistream dump used by metadata index / corpus specs.
+# Stream 1 holds articles (ns 0), stream 2 holds category pages (ns 14).
+module MultistreamFixture
+  def page_xml(id:, ns:, title:, text:)
+    escaped = text.gsub("&", "&amp;").gsub("<", "&lt;").gsub(">", "&gt;")
+    <<~XML
+      <page>
+        <title>#{title}</title>
+        <ns>#{ns}</ns>
+        <id>#{id}</id>
+        <revision>
+          <id>#{id * 100}</id>
+          <text bytes="#{text.bytesize}">#{escaped}</text>
+        </revision>
+      </page>
+    XML
+  end
+
+  def bzip2(data)
+    out, status = Open3.capture2("bzip2", "-c", stdin_data: data)
+    raise "bzip2 failed" unless status.success?
+
+    out
+  end
+
+  # @return [Array(String, String)] [multistream_path, index_path]
+  def create_fixture(dir)
+    stream1_pages = [
+      page_xml(id: 1, ns: 0, title: "Film A",
+               text: "Intro.\n== Plot ==\nStory here.\n== Reception ==\nGood.\n[[Category:Japanese films]]\n"),
+      page_xml(id: 2, ns: 0, title: "Film B",
+               text: "Intro.\n== Synopsis ==\nStory here.\n[[Category:French films|B]]\n"),
+      page_xml(id: 3, ns: 0, title: "Person X",
+               text: "Bio.\n== Career == <!-- legacy anchor -->\nActing.\n== '''Style''' ==\nDistinct prose.\n<!-- [[Category:Hidden]] -->\n[[Category:Japanese actors]]\n"),
+      page_xml(id: 4, ns: 0, title: "Old Film",
+               text: "#REDIRECT [[Film A]]\n[[Category:Japanese films]]\n")
+    ]
+    stream2_pages = [
+      page_xml(id: 5, ns: 14, title: "Category:Japanese films", text: "[[Category:Films]]\n"),
+      page_xml(id: 6, ns: 14, title: "Category:French films", text: "[[Category:Films]]\n"),
+      page_xml(id: 7, ns: 14, title: "Category:Films", text: "Top category.\n"),
+      page_xml(id: 8, ns: 14, title: "Category:Japanese actors", text: "[[Category:People]]\n")
+    ]
+
+    stream1 = bzip2(stream1_pages.join)
+    stream2 = bzip2(stream2_pages.join)
+
+    multistream_path = File.join(dir, "testwiki-20260101-pages-articles-multistream.xml.bz2")
+    File.binwrite(multistream_path, stream1 + stream2)
+
+    offset2 = stream1.bytesize
+    index_lines = [
+      "0:1:Film A", "0:2:Film B", "0:3:Person X", "0:4:Old Film",
+      "#{offset2}:5:Category:Japanese films", "#{offset2}:6:Category:French films",
+      "#{offset2}:7:Category:Films", "#{offset2}:8:Category:Japanese actors"
+    ]
+    index_path = File.join(dir, "testwiki-20260101-pages-articles-multistream-index.txt")
+    File.write(index_path, index_lines.join("\n") + "\n")
+
+    [multistream_path, index_path]
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: wp2txt
 version: !ruby/object:Gem::Version
-  version: 2.1.1
+  version: 2.2.0
 platform: ruby
 authors:
 - Yoichiro Hasebe
@@ -197,10 +197,12 @@ email:
 - yohasebe@gmail.com
 executables:
 - wp2txt
+- wp2txt-mcp
 extensions: []
 extra_rdoc_files: []
 files:
 - ".dockerignore"
+- ".github/FUNDING.yml"
 - ".github/workflows/ci.yml"
 - ".gitignore"
 - ".solargraph.yml"
@@ -214,6 +216,7 @@ files:
 - README_ja.md
 - Rakefile
 - bin/wp2txt
+- bin/wp2txt-mcp
 - image/wp2txt-logo.svg
 - image/wp2txt.svg
 - lib/wp2txt.rb
@@ -224,6 +227,8 @@ files:
 - lib/wp2txt/cli_ui.rb
 - lib/wp2txt/config.rb
 - lib/wp2txt/constants.rb
+- lib/wp2txt/corpus.rb
+- lib/wp2txt/corpus_jobs.rb
 - lib/wp2txt/data/html_entities.json
 - lib/wp2txt/data/language_metadata.json
 - lib/wp2txt/data/language_tiers.json
@@ -233,10 +238,13 @@ files:
 - lib/wp2txt/extractor.rb
 - lib/wp2txt/file_utils.rb
 - lib/wp2txt/formatter.rb
+- lib/wp2txt/fts_index.rb
 - lib/wp2txt/global_data_cache.rb
 - lib/wp2txt/index_cache.rb
+- lib/wp2txt/index_commands.rb
 - lib/wp2txt/magic_words.rb
 - lib/wp2txt/memory_monitor.rb
+- lib/wp2txt/metadata_index.rb
 - lib/wp2txt/multistream.rb
 - lib/wp2txt/output_writer.rb
 - lib/wp2txt/parser_functions.rb
@@ -264,15 +272,18 @@ files:
 - spec/cli_spec.rb
 - spec/config_spec.rb
 - spec/constants_spec.rb
+- spec/corpus_spec.rb
 - spec/file_utils_spec.rb
 - spec/fixtures/samples.rb
 - spec/formatter_sections_spec.rb
+- spec/fts_index_spec.rb
 - spec/global_data_cache_spec.rb
 - spec/index_cache_spec.rb
 - spec/integration_spec.rb
 - spec/magic_words_spec.rb
 - spec/markers_spec.rb
 - spec/memory_monitor_spec.rb
+- spec/metadata_index_spec.rb
 - spec/multistream_spec.rb
 - spec/output_writer_spec.rb
 - spec/parser_functions_spec.rb
@@ -281,6 +292,7 @@ files:
 - spec/section_extractor_spec.rb
 - spec/spec_helper.rb
 - spec/stream_processor_spec.rb
+- spec/support/multistream_fixture.rb
 - spec/template_data_spec.rb
 - spec/template_expander_spec.rb
 - spec/template_processing_spec.rb
@@ -321,15 +333,18 @@ test_files:
 - spec/cli_spec.rb
 - spec/config_spec.rb
 - spec/constants_spec.rb
+- spec/corpus_spec.rb
 - spec/file_utils_spec.rb
 - spec/fixtures/samples.rb
 - spec/formatter_sections_spec.rb
+- spec/fts_index_spec.rb
 - spec/global_data_cache_spec.rb
 - spec/index_cache_spec.rb
 - spec/integration_spec.rb
 - spec/magic_words_spec.rb
 - spec/markers_spec.rb
 - spec/memory_monitor_spec.rb
+- spec/metadata_index_spec.rb
 - spec/multistream_spec.rb
 - spec/output_writer_spec.rb
 - spec/parser_functions_spec.rb
@@ -338,6 +353,7 @@ test_files:
 - spec/section_extractor_spec.rb
 - spec/spec_helper.rb
 - spec/stream_processor_spec.rb
+- spec/support/multistream_fixture.rb
 - spec/template_data_spec.rb
 - spec/template_expander_spec.rb
 - spec/template_processing_spec.rb