wp2txt 2.3.2 → 2.3.3

data/lib/wp2txt/corpus.rb

@@ -14,6 +14,7 @@ require_relative "metadata_index"
 require_relative "fts_index"
 require_relative "section_extractor"
 require_relative "version"
+require_relative "output_path"
 
 module Wp2txt
   # Facade over a local dump: single-article access (Tier 0, multistream),
@@ -324,7 +325,7 @@ module Wp2txt
                        title_match: nil, limit: 0, titles: nil,
                        chunk_size: nil, chunk_overlap: 0,
                        max_articles: DEFAULT_MAX_SYNC_ARTICLES, num_processes: 4,
-                       progress: nil, cancel_check: nil)
+                       progress: nil, cancel_check: nil, overwrite: false)
       if content == "sections" && Array(sections).empty? && alias_set.nil?
         raise ArgumentError, "content: \"sections\" requires sections or alias_set"
       end
@@ -394,46 +395,48 @@ module Wp2txt
       titles_done = 0
       sample = []
 
-      File.open(output_path, "w") do |f|
-        titles.each_slice(EXTRACT_BATCH_SIZE) do |batch|
-          raise Cancelled if cancel_check&.call
-
-          pages = reader.extract_articles_parallel(batch, num_processes: num_processes)
-          batch.each do |t|
-            page = pages[t]
-            next unless page
-
-            records = build_records(page, content, resolved_sections, chunk_size, chunk_overlap)
-            next if records.empty?
-
-            articles_extracted += 1
-            records.each do |record|
-              f.puts(JSON.generate(record))
-              records_written += 1
-              sample << record if sample.size < 3
+      meta_path = "#{output_path}.meta.json"
+      OutputPath.write_pair(output_path, overwrite: overwrite) do |staged_output, staged_meta|
+        File.open(staged_output, "w") do |f|
+          titles.each_slice(EXTRACT_BATCH_SIZE) do |batch|
+            raise Cancelled if cancel_check&.call
+
+            pages = reader.extract_articles_parallel(batch, num_processes: num_processes)
+            batch.each do |t|
+              page = pages[t]
+              next unless page
+
+              records = build_records(page, content, resolved_sections, chunk_size, chunk_overlap)
+              next if records.empty?
+
+              articles_extracted += 1
+              records.each do |record|
+                f.puts(JSON.generate(record))
+                records_written += 1
+                sample << record if sample.size < 3
+              end
             end
+            titles_done += batch.size
+            progress&.call(titles_done, titles.size)
           end
-          titles_done += batch.size
-          progress&.call(titles_done, titles.size)
         end
-      end
 
-      meta_path = "#{output_path}.meta.json"
-      File.write(meta_path, JSON.pretty_generate(
-        tool: "wp2txt #{Wp2txt::VERSION}",
-        dump: dump_name,
-        generated_at: Time.now.utc.iso8601,
-        query: (titles_record || filters.compact).merge(
-          content: content, resolved_sections: resolved_sections,
-          chunk_size: chunk_size, chunk_overlap: chunk_size ? chunk_overlap : nil
-        ).compact,
-        alias_set_contents: alias_contents,
-        total_matching: total,
-        articles_extracted: articles_extracted,
-        records_written: records_written,
-        truncated: truncated,
-        not_found: not_found
-      ))
+        File.write(staged_meta, JSON.pretty_generate(
+          tool: "wp2txt #{Wp2txt::VERSION}",
+          dump: dump_name,
+          generated_at: Time.now.utc.iso8601,
+          query: (titles_record || filters.compact).merge(
+            content: content, resolved_sections: resolved_sections,
+            chunk_size: chunk_size, chunk_overlap: chunk_size ? chunk_overlap : nil
+          ).compact,
+          alias_set_contents: alias_contents,
+          total_matching: total,
+          articles_extracted: articles_extracted,
+          records_written: records_written,
+          truncated: truncated,
+          not_found: not_found
+        ))
+      end
 
       { output_path: output_path, meta_path: meta_path, dump: dump_name,
         total_matching: total, articles_extracted: articles_extracted,
@@ -661,15 +664,19 @@ module Wp2txt
     def run_sql_on(db, sql, limit)
       columns = nil
       rows = []
+      truncated = false
       db.query(sql) do |result|
         columns = result.columns
         result.each do |row|
-          break if rows.size >= limit
+          if rows.size >= limit
+            truncated = true
+            break
+          end
 
           rows << row.map { |v| v.is_a?(String) && v.length > SQL_CELL_LIMIT ? "#{v[0, SQL_CELL_LIMIT]}…" : v }
         end
       end
-      { columns: columns, rows: rows, row_count: rows.size, truncated: rows.size >= limit }
+      { columns: columns, rows: rows, row_count: rows.size, truncated: truncated }
     end
 
     # Execute the query in a forked child with a hard deadline: the child opens
@@ -727,38 +734,24 @@ module Wp2txt
       end
     end
 
-    # Write the full query result to output_path as JSONL. Atomicity: the
-    # child (or inline fallback) writes "#{output_path}.partial"; the parent
-    # renames it into place only on success and removes it on every failure
-    # path (child crash, timeout kill, error over the pipe) — a partially
-    # written file is never presented as a result. The .meta.json sidecar is
-    # written by the parent after the rename succeeds.
+    # Stage JSONL and provenance in unique files, then publish on success.
+    # OutputPath owns exclusive destination reservations and failure cleanup.
     def query_sql_to_file(sql, timeout, attachments, output_path, overwrite)
-      if File.exist?(output_path) && !overwrite
-        raise ArgumentError, "output file already exists: #{output_path} (pass overwrite: true to replace it)"
-      end
-
-      partial = "#{output_path}.partial"
-      FileUtils.rm_f(partial)
-      outcome = begin
-        if Process.respond_to?(:fork)
-          run_sql_file_in_subprocess(sql, timeout, attachments, partial)
-        else
-          db = build_readonly_connection(attach_fts: fts.built?, attachments: attachments)
-          begin
-            run_sql_file_on(db, sql, partial)
-          ensure
-            db.close
-          end
-        end
-      rescue StandardError
-        FileUtils.rm_f(partial)
-        raise
+      outcome = OutputPath.write_pair(output_path, overwrite: overwrite) do |partial, staged_meta|
+        result = if Process.respond_to?(:fork)
+                   run_sql_file_in_subprocess(sql, timeout, attachments, partial)
+                 else
+                   db = build_readonly_connection(attach_fts: fts.built?, attachments: attachments)
+                   begin
+                     run_sql_file_on(db, sql, partial)
+                   ensure
+                     db.close
+                   end
+                 end
+        write_sql_sidecar(staged_meta, sql, attachments, result)
+        result
       end
 
-      File.rename(partial, output_path)
-      write_sql_sidecar(output_path, sql, attachments, outcome)
-
       result = { output_path: output_path, meta_path: "#{output_path}.meta.json",
                  columns: outcome[:columns], row_count: outcome[:row_count],
                  truncated: outcome[:truncated], cells_clipped: outcome[:cells_clipped],
@@ -772,6 +765,7 @@ module Wp2txt
     # subprocess path: the 30s SIGKILL deadline covers the writing too.
     def run_sql_file_on(db, sql, partial_path)
       columns = nil
+      column_mapping = nil
       row_count = 0
       cells_clipped = 0
       truncated = false
@@ -779,7 +773,11 @@ module Wp2txt
 
       File.open(partial_path, "w") do |f|
         db.query(sql) do |result|
-          columns = unique_columns(result.columns)
+          original_columns = result.columns
+          columns = unique_columns(original_columns)
+          column_mapping = original_columns.each_with_index.map do |name, ordinal|
+            { ordinal: ordinal, original_name: name, output_name: columns[ordinal] }
+          end
           result.each do |row|
             if row_count >= SQL_FILE_ROW_LIMIT
               truncated = true
@@ -788,8 +786,8 @@ module Wp2txt
 
             record = {}
             row.each_with_index do |value, i|
-              if value.is_a?(String) && value.length > SQL_FILE_CELL_LIMIT
-                value = "#{value[0, SQL_FILE_CELL_LIMIT]}…"
+              if value.is_a?(String) && value.bytesize > SQL_FILE_CELL_LIMIT
+                value = clip_file_cell(value)
                 cells_clipped += 1
               end
               record[columns[i]] = value
@@ -801,20 +799,37 @@ module Wp2txt
         end
       end
 
-      { columns: columns, row_count: row_count, truncated: truncated,
+      { columns: columns, column_mapping: column_mapping, row_count: row_count, truncated: truncated,
         cells_clipped: cells_clipped, sample: sample }
     end
 
     # Duplicate result column names (SELECT 1 AS x, 2 AS x) are suffixed
     # (_2, _3, ...) so every JSONL record key is unique
     def unique_columns(columns)
-      seen = Hash.new(0)
-      columns.map do |c|
-        seen[c] += 1
-        seen[c] == 1 ? c : "#{c}_#{seen[c]}"
+      reserved = columns.to_h { |name| [name, true] }
+      used = {}
+      columns.map do |name|
+        candidate = name
+        suffix = 2
+        while used[candidate] || (candidate != name && reserved[candidate])
+          candidate = "#{name}_#{suffix}"
+          suffix += 1
+        end
+        used[candidate] = true
+        candidate
       end
     end
 
+    # SQL_FILE_CELL_LIMIT is a byte ceiling INCLUDING the UTF-8 ellipsis.
+    # Remove only the incomplete UTF-8 suffix after a byte-based cut.
+    def clip_file_cell(value)
+      utf8 = value.dup.force_encoding(Encoding::UTF_8)
+      raise JSON::GeneratorError, "SQL cell contains invalid UTF-8" unless utf8.valid_encoding?
+
+      prefix = utf8.byteslice(0, SQL_FILE_CELL_LIMIT - "…".bytesize).force_encoding(Encoding::UTF_8)
+      "#{prefix.scrub("")}…"
+    end
+
     # Subprocess driver for file-output mode; same fork/pipe/SIGKILL
     # structure as run_sql_in_subprocess, but the child writes the rows to
     # partial_path and ships back only the summary
@@ -856,9 +871,9 @@ module Wp2txt
       reader_io&.close
     end
 
-    # Reproducibility sidecar, written by the parent after the atomic rename
-    def write_sql_sidecar(output_path, sql, attachments, outcome)
-      File.write("#{output_path}.meta.json", JSON.pretty_generate(
+    # Reproducibility sidecar, staged by the parent before publication.
+    def write_sql_sidecar(meta_path, sql, attachments, outcome)
+      File.write(meta_path, JSON.pretty_generate(
         tool: "query_sql",
         dump: dump_name,
         built_with: @metadata.stats&.dig(:built_with),
@@ -867,6 +882,8 @@ module Wp2txt
         row_count: outcome[:row_count],
         truncated: outcome[:truncated],
         cells_clipped: outcome[:cells_clipped],
+        column_mapping: outcome[:column_mapping],
+        cell_byte_limit: SQL_FILE_CELL_LIMIT,
         generated_at: Time.now.utc.iso8601,
         wp2txt_version: Wp2txt::VERSION
       ))

data/lib/wp2txt/corpus_jobs.rb

@@ -22,22 +22,24 @@ module Wp2txt
     # unbounded concurrent jobs would multiply workers against the same dump.
     # @return [Hash] { job_id:, status: "running" } or { error: ... }
     def start_extract(params)
-      running = @mutex.synchronize { @jobs.values.find { |s| s[:status] == "running" } }
-      if running
-        return { error: "another job is already running (#{running[:job_id]}); " \
-                        "poll job_status or cancel_job before starting a new one" }
+      job_id = @mutex.synchronize do
+        running = @jobs.values.find { |state| state[:status] == "running" }
+        if running
+          return { error: "another job is already running (#{running[:job_id]}); " \
+                          "poll job_status or cancel_job before starting a new one" }
+        end
+        id = format("job-%04d", @seq += 1)
+        @jobs[id] = {
+          job_id: id, status: "running", started_at: Time.now.utc.iso8601,
+          params: params, titles_done: 0, titles_total: nil, cancel: false
+        }
+        id
       end
 
-      job_id = @mutex.synchronize { format("job-%04d", @seq += 1) }
-      state = {
-        job_id: job_id, status: "running", started_at: Time.now.utc.iso8601,
-        params: params, titles_done: 0, titles_total: nil, cancel: false
-      }
-      @mutex.synchronize { @jobs[job_id] = state }
-
       thread = Thread.new do
-        corpus = @factory.call
+        corpus = nil
         begin
+          corpus = @factory.call
           result = corpus.extract_corpus(
             **params,
             max_articles: nil,
@@ -52,7 +54,7 @@ module Wp2txt
         rescue StandardError => e
           update(job_id) { |s| s[:status] = "error"; s[:error] = "#{e.class}: #{e.message}"; s[:finished_at] = Time.now.utc.iso8601 }
         ensure
-          corpus.close
+          corpus&.close
         end
       end
       thread.report_on_exception = false

data/lib/wp2txt/fts_index.rb

@@ -19,6 +19,12 @@ module Wp2txt
   # Queries ATTACH the Tier 1 metadata DB so category/section/redirect
   # filters compose with MATCH in plain SQL.
   class FtsIndex
+    class ShortQueryError < ArgumentError
+      def code
+        "query_too_short"
+      end
+    end
+
     SCHEMA_VERSION = 2
     CACHE_SUFFIX = "_fts.sqlite3"
 
@@ -206,6 +212,9 @@ module Wp2txt
     # @return [Hash] { total:, total_is_capped:, hits: [{page_id:, title:, heading:, ord:}] }
     def search(query, mode: "phrase", sections: nil, category: nil, depth: 0,
                limit: 20, offset: 0, count: "capped", count_cap: 1000)
+      if mode == "phrase" && tokenizer == "trigram" && query.length < 3
+        raise ShortQueryError, "trigram phrase searches require at least 3 Unicode characters"
+      end
       match_expr = mode == "query" ? query : phrase_query(query)
 
       conds = ["fts_sections MATCH ?", "p.namespace = 0", "p.redirect_to IS NULL"]
@@ -386,10 +395,10 @@ module Wp2txt
       batches = pairs.each_slice(STREAMS_PER_BATCH).to_a
       done = 0
 
-      Parallel.map(
+      Parallel.each(
         batches,
         in_processes: @num_processes,
-        finish: lambda { |_item, _idx, rows|
+                finish: lambda { |_item, _idx, rows|
           index.insert_batch(rows)
           done += 1
           progress&.call(done, batches.size)
@@ -423,7 +432,7 @@ module Wp2txt
       title = block[MetadataIndexBuilder::TITLE_REGEX, 1]
       return unless title && !title.empty?
 
-      ns = (block[MetadataIndexBuilder::NS_REGEX, 1] || "0").to_i
+      ns = Wp2txt.namespace_id(block[MetadataIndexBuilder::NS_REGEX, 1])
       return unless ns.zero?
 
       page_id = block[MetadataIndexBuilder::ID_REGEX, 1]&.to_i

data/lib/wp2txt/metadata_index.rb

@@ -640,10 +640,10 @@ module Wp2txt
       batches = pairs.each_slice(STREAMS_PER_BATCH).to_a
       done = 0
 
-      Parallel.map(
+      Parallel.each(
         batches,
         in_processes: @num_processes,
-        finish: lambda { |_item, _idx, result|
+                finish: lambda { |_item, _idx, result|
           index.insert_batch(result)
           done += 1
           progress&.call(done, batches.size)
@@ -686,7 +686,7 @@ module Wp2txt
       return unless page_id
 
       title = unescape_xml(title)
-      ns = (block[NS_REGEX, 1] || "0").to_i
+      ns = Wp2txt.namespace_id(block[NS_REGEX, 1])
       text = block[TEXT_REGEX, 1] || ""
       text = unescape_xml(text)
       # Strip HTML comments before scanning, matching what the Article parser

data/lib/wp2txt/output_path.rb

@@ -1,27 +1,87 @@
 # frozen_string_literal: true
 
+require "tempfile"
+require "tmpdir"
+
 module Wp2txt
-  # Server-side output path confinement shared by the file-writing tools
-  # (extract_corpus / start_extract_job / query_sql with output_path).
-  # Paths must resolve under the server's output directory — an agent mixing
-  # up paths must not be able to clobber arbitrary user files — and existing
-  # files are not replaced unless overwrite is set.
+  # Output directories are dedicated, trusted directories. Reject links below
+  # the trusted platform temp root (whose ancestors may be OS aliases on macOS).
+  # This does not defend against an attacker replacing parent directories.
   module OutputPath
     module_function
 
-    # @return [String] the confined, absolute output path
-    # @raise [ArgumentError] when the path escapes base_dir or the file exists
+    def reject_symlinks!(path)
+      current = File.expand_path(path)
+      temp_roots = [File.expand_path(Dir.tmpdir), File.realpath(Dir.tmpdir)]
+      loop do
+        raise ArgumentError, "symbolic links are not allowed in output paths: #{current}" if File.symlink?(current)
+        parent = File.dirname(current)
+        break if parent == current || temp_roots.include?(current)
+
+        current = parent
+      end
+    end
+
+    def validate_pair!(path, overwrite: false)
+      [path, "#{path}.meta.json"].each do |destination|
+        reject_symlinks!(destination)
+        if File.exist?(destination) && !overwrite
+          raise ArgumentError, "output file already exists: #{destination} (pass overwrite: true to replace it)"
+        end
+        if File.exist?(destination) && !File.file?(destination)
+          raise ArgumentError, "output destination must be a file: #{destination}"
+        end
+      end
+    end
+
+    # Return the confined absolute path; the writer repeats validation and
+    # reserves both destinations with EXCL to close the check/create race.
     def confine(output_path, base_dir, overwrite: false)
       path = File.expand_path(output_path, base_dir)
       base = File.expand_path(base_dir)
-      unless path == base || path.start_with?(base + File::SEPARATOR)
+      unless path.start_with?(base + File::SEPARATOR)
         raise ArgumentError, "output_path must stay within the server output directory (#{base})"
       end
-      if File.exist?(path) && !overwrite
-        raise ArgumentError, "output file already exists: #{path} (pass overwrite: true to replace it)"
-      end
-
+      validate_pair!(path, overwrite: overwrite)
       path
     end
+
+    # Reserve destinations, stage both files, then publish with rename. EXCL
+    # reservations are empty until publication; the pair is not a transaction.
+    # Failure removes only reservations owned by this call, never prior output.
+    def write_pair(path, overwrite: false)
+      validate_pair!(path, overwrite: overwrite)
+      destinations = [path, "#{path}.meta.json"]
+      reservations = {}
+      temporary = []
+      begin
+        unless overwrite
+          destinations.each do |destination|
+            File.open(destination, File::WRONLY | File::CREAT | File::EXCL, 0o600) do |file|
+              reservations[destination] = file.stat
+            end
+          end
+        end
+        destinations.each do |destination|
+          temporary << Tempfile.create([".wp2txt-", ".partial"], File.dirname(destination))
+          temporary.last.close
+        end
+        result = yield(*temporary.map(&:path))
+        destinations.each_with_index do |destination, index|
+          reject_symlinks!(destination)
+          File.rename(temporary[index].path, destination)
+          reservations.delete(destination)
+        end
+        result
+      rescue Errno::EEXIST => e
+        raise ArgumentError, "output file already exists: #{e.message} (pass overwrite: true to replace it)"
+      ensure
+        temporary.each { |file| File.unlink(file.path) if File.exist?(file.path) }
+        reservations.each do |destination, stat|
+          current = File.lstat(destination) rescue nil
+          File.unlink(destination) if current && current.dev == stat.dev && current.ino == stat.ino
+        end
+      end
+    end
   end
 end

data/lib/wp2txt/stream_processor.rb

@@ -23,6 +23,7 @@ module Wp2txt
       @input_path = input_path
       @bz2_gem = bz2_gem
       @buffer = +""
+      @pending_bytes = +"".b
       @file_pointer = nil
       @adaptive_buffer = adaptive_buffer
       @buffer_size = adaptive_buffer ? calculate_optimal_buffer_size : DEFAULT_BUFFER_SIZE
@@ -99,7 +100,8 @@ module Wp2txt
     # Process a single XML file
     def process_xml_file(xml_file)
       @buffer = +""
-      @file_pointer = File.open(xml_file, "r:UTF-8")
+      @pending_bytes = +"".b
+      @file_pointer = File.open(xml_file, "rb")
 
       while (page = extract_next_page)
         result = parse_page_xml(page)
@@ -120,6 +122,7 @@ module Wp2txt
       end
 
       @buffer = +""
+      @pending_bytes = +"".b
       @file_pointer = open_bz2_stream
 
       while (page = extract_next_page)
@@ -158,14 +161,21 @@ module Wp2txt
     # Fill buffer from file pointer
     def fill_buffer
       chunk = @file_pointer.read(@buffer_size)
-      return false unless chunk
+      unless chunk
+        @buffer << @pending_bytes.to_s.dup.force_encoding(Encoding::UTF_8).scrub("")
+        @pending_bytes = +"".b
+        return false
+      end
 
       @bytes_read += chunk.bytesize
-
-      # Handle encoding for bz2 streams
-      chunk = chunk.force_encoding("UTF-8")
-      chunk = chunk.scrub("")
-      @buffer << chunk
+      bytes = @pending_bytes.to_s.b + chunk.b
+      # Retain a trailing UTF-8 sequence until the next read. Only complete
+      # chunks are scrubbed, so valid characters split by read are preserved.
+      tail = bytes[/[\xC2-\xF4][\x80-\xBF]{0,2}\z/n]
+      width = tail && (tail.getbyte(0) < 0xE0 ? 2 : tail.getbyte(0) < 0xF0 ? 3 : 4)
+      @pending_bytes = tail && tail.bytesize < width ? tail : +"".b
+      bytes = bytes.byteslice(0, bytes.bytesize - @pending_bytes.bytesize)
+      @buffer << bytes.force_encoding(Encoding::UTF_8).scrub("")
 
       # Adaptive buffer adjustment: if memory is low, reduce buffer size
       if @adaptive_buffer && MemoryMonitor.memory_low?
@@ -223,8 +233,8 @@ module Wp2txt
       return nil unless title_node
 
       title = title_node.content
-      # Skip special pages (containing colon in title like "Wikipedia:", "File:", etc.)
-      return nil if title.include?(":")
+      namespace = title_node.parent.at_css("ns")&.text
+      return nil unless Wp2txt.namespace_id(namespace).zero?
 
       text = text_node.content
 

data/lib/wp2txt/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Wp2txt
-  VERSION = "2.3.2"
+  VERSION = "2.3.3"
 end

data/spec/fts_index_spec.rb

@@ -171,6 +171,46 @@ RSpec.describe "Wp2txt Full-Text Search" do
         end
       end
 
+      ["", "東", "東京", "é", "😀"].each do |query|
+        it "rejects the short phrase #{query.inspect} with a stable code" do
+          expect { @fts.search(query, count: "exact") }.to raise_error(Wp2txt::FtsIndex::ShortQueryError) { |error|
+            expect(error.code).to eq("query_too_short")
+          }
+        end
+      end
+
+      it "leaves raw FTS syntax to SQLite even for short expressions" do
+        expect(@fts.search('"東"', mode: "query", count: "exact")[:total]).to eq(0)
+        expect(@fts.search("\xFF".b, mode: "query", count: "exact")[:total]).to eq(0)
+      end
+
+      it "accepts three Unicode characters" do
+        expect(@fts.search("東京都", count: "exact")[:total]).to eq(0)
+      end
+
+      it "returns a coded MCP tool error instead of a zero-result success" do
+        @fts.close
+        requests = [
+          { jsonrpc: "2.0", id: 1, method: "initialize", params: {
+            protocolVersion: "2025-03-26", capabilities: {}, clientInfo: { name: "regression", version: "1" }
+          } },
+          { jsonrpc: "2.0", method: "notifications/initialized" },
+          { jsonrpc: "2.0", id: 2, method: "tools/call", params: {
+            name: "search_text", arguments: { query: "東京", count: "exact" }
+          } }
+        ]
+        stdout, stderr, status = Open3.capture3(RbConfig.ruby,
+          File.expand_path("../bin/wp2txt-mcp", __dir__), "--input", @multistream_path,
+          "--cache-dir", File.dirname(@multistream_path),
+          stdin_data: requests.map { |request| JSON.generate(request) }.join("\n") + "\n")
+        expect(status.success?).to be(true), stderr
+        response = stdout.lines.map { |line| JSON.parse(line) }.find { |message| message["id"] == 2 }
+        expect(response.dig("result", "isError")).to be true
+        payload = JSON.parse(response.dig("result", "content", 0, "text"))
+        expect(payload["code"]).to eq("query_too_short")
+        expect(payload).not_to have_key("total")
+      end
+
       it "matches substrings of three or more characters" do
         result = @fts.search("tory", count: "exact")
         expect(result[:total]).to eq(2)