wp2txt 2.3.2 → 2.3.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 874d5f0c6463fc612080339cbbcbe560b51aba31a8a5e9478d95719c53744051
-  data.tar.gz: e8377c0df46cd7d35ca8a85cea4f52e6dc4ca57330004ce5fc6f11913aa1abcf
+  metadata.gz: 051b5501ecf2aad35ca972af6527c5afbce88222c17e9c93dc76f806b91a8e9d
+  data.tar.gz: 02efabb7102da40c144a3c8204f564a8a40dfa1e77a54813e40bd6740693a264
 SHA512:
-  metadata.gz: ce2ca7660379c37a3f36d6d731aa3a9c1798998c10dfb5ff012503f571653b26aef6324e579c1a8af7f0d2ff52cf90fc8c6a50f44f21d40b73d132277cb50a26
-  data.tar.gz: d8197213c46510742d015dcea77c8ff5c40d48dac7beff8cc409ce1e4ce5f3e138e47d7df9ce675a0614785b21bb266664f8c082c94080c4f3a29c712ffeab07
+  metadata.gz: d89ff3e231200c36f647284fee43565021c69c73ac8d9f23ca42b58008d2d0d3ac5598fe63aa3736a45ce933622dd28f649f721635a63e782d638f444602d08b
+  data.tar.gz: c70fccdb332688f805e77dbefd291563045e2f53c07e7cc994dbff0a7f8858dded28f18bce4bcb1296e7594f12ee54eea12a4573beee04a8944d4e39b7d35620

data/.dockerignore

@@ -15,6 +15,9 @@ scripts
 .solargraph.yml
 .rubocop.yml
 Gemfile.lock
+.private-doc-tokens
+**/.DS_Store
+.ruby-version
 CLAUDE.md
 DEVELOPMENT.md
 DEVELOPMENT_ja.md

data/.github/workflows/publish-image.yml

@@ -0,0 +1,140 @@
+name: Publish image
+
+# Publishes the multi-arch container image to the GitHub Container Registry.
+#
+# Publishing happens here rather than from a maintainer's machine on purpose. A
+# local build sends the working tree as the build context, so anything sitting
+# there untracked rides along; a runner starts from a clean checkout, where such
+# files do not exist. Twice, private files reached published images this way.
+#
+# Fires on a v* tag. Run it by hand from the Actions tab for a dry run: the
+# default leaves `push` off, so it builds and gates without publishing.
+
+on:
+  push:
+    tags: ['v*']
+  workflow_dispatch:
+    inputs:
+      push:
+        description: 'Publish to GHCR (off = build and gate only)'
+        type: boolean
+        default: false
+
+env:
+  IMAGE: ghcr.io/yohasebe/wp2txt
+
+jobs:
+  publish:
+    name: Build, gate, publish
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Work out the version and whether to publish
+        id: plan
+        run: |
+          set -euo pipefail
+          file_version=$(grep -oE '[0-9]+\.[0-9]+\.[0-9]+' lib/wp2txt/version.rb | head -1)
+          if [ -z "$file_version" ]; then
+            echo "could not read a version from lib/wp2txt/version.rb" >&2
+            exit 1
+          fi
+
+          if [ "${GITHUB_EVENT_NAME}" = "push" ]; then
+            tag_version="${GITHUB_REF_NAME#v}"
+            if [ "$tag_version" != "$file_version" ]; then
+              echo "tag ${GITHUB_REF_NAME} disagrees with lib/wp2txt/version.rb (${file_version})" >&2
+              echo "Publishing would tag the image with a version the code does not claim." >&2
+              exit 1
+            fi
+            publish=true
+          else
+            publish="${{ inputs.push }}"
+          fi
+
+          echo "version=$file_version" >> "$GITHUB_OUTPUT"
+          echo "publish=$publish" >> "$GITHUB_OUTPUT"
+          echo "Version $file_version; publish=$publish"
+
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '3.3'
+
+      - uses: docker/setup-qemu-action@v3
+
+      - uses: docker/setup-buildx-action@v3
+
+      # Single arch, loaded into the daemon, so the gate can open it. The
+      # multi-arch build below reuses these layers from the builder's cache.
+      - name: Build for inspection (linux/amd64)
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          platforms: linux/amd64
+          load: true
+          push: false
+          tags: ${{ env.IMAGE }}:gate-candidate
+
+      - name: Gate — compare the image against the build context
+        run: ruby scripts/verify_image.rb "${IMAGE}:gate-candidate"
+
+      - name: Log in to GHCR
+        if: steps.plan.outputs.publish == 'true'
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push (linux/amd64, linux/arm64)
+        id: publish
+        if: steps.plan.outputs.publish == 'true'
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: |
+            ${{ env.IMAGE }}:${{ steps.plan.outputs.version }}
+            ${{ env.IMAGE }}:latest
+          labels: |
+            org.opencontainers.image.source=https://github.com/yohasebe/wp2txt
+            org.opencontainers.image.version=${{ steps.plan.outputs.version }}
+
+      # Reading it back without credentials is the only proof that a user can
+      # pull what was just published.
+      - name: Check the published digest is readable anonymously
+        if: steps.plan.outputs.publish == 'true'
+        run: |
+          set -euo pipefail
+          digest="${{ steps.publish.outputs.digest }}"
+          docker logout ghcr.io
+          for attempt in 1 2 3 4 5; do
+            if docker manifest inspect "${IMAGE}@${digest}" > /dev/null 2>&1; then
+              echo "readable anonymously: ${IMAGE}@${digest}"
+              exit 0
+            fi
+            echo "attempt ${attempt} failed; ghcr.io can lag a few seconds, retrying in 10s"
+            sleep 10
+          done
+          echo "${IMAGE}@${digest} is not readable anonymously" >&2
+          exit 1
+
+      - name: Summary
+        if: always()
+        run: |
+          {
+            echo "### wp2txt image"
+            echo
+            echo "- version: \`${{ steps.plan.outputs.version }}\`"
+            echo "- published: \`${{ steps.plan.outputs.publish }}\`"
+            if [ -n "${{ steps.publish.outputs.digest }}" ]; then
+              echo "- digest: \`${{ steps.publish.outputs.digest }}\`"
+              echo "- pull: \`docker pull ${IMAGE}:${{ steps.plan.outputs.version }}\`"
+            fi
+            echo "- run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+          } >> "$GITHUB_STEP_SUMMARY"

data/CHANGELOG.md

@@ -8,6 +8,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 Entries were rewritten in August 2026 to describe what changed for people using
 wp2txt, rather than how it was implemented. The changes themselves are unaltered.
 
+## [2.3.3] - 2026-09-08
+
+- **Short searches no longer report a false zero**: in Japanese, Chinese, and Korean indexes a phrase of one or two characters could match nothing and come back as `0 matches`, which reads exactly like a term that is genuinely absent from the dump. Such a search now fails with an explicit error instead. If you recorded a zero result for a short term with an earlier version, re-check it
+- **Characters no longer vanish from extracted text**: a multi-byte character that happened to straddle an internal read boundary was dropped, silently and at unpredictable positions. Any non-ASCII text could be affected. Re-extract if you need the output to be exact
+- **Articles whose title contains a colon are no longer skipped**: `Star Trek: Voyager` and titles like it were mistaken for administrative pages and left out of both extraction and indexes. Rebuild with `--build-index -U` and re-extract to pick them up
+- **`query_sql` no longer loses values when column names repeat**: a query selecting two columns under the same name kept only one of them. Names are now made unique, and the `.meta.json` beside a file result records which output name corresponds to which column of your query
+- **Cell clipping counts bytes, not characters**: a long cell was measured in characters, so it could exceed the limit and still be reported as unclipped — 30,000 Japanese characters are 90,000 bytes. The limit is now 65,536 bytes, cut at a character boundary, and the count of clipped cells is accurate
+- **A result of exactly the requested size is no longer marked truncated**: `truncated` came back true when a result happened to fill the limit exactly, with nothing left behind
+- **Output files stay inside the output directory**: a symbolic link within it could redirect a write outside, the `.meta.json` sidecar was not checked at all, and an existing file could slip past `overwrite: false`. Output and sidecar are now reserved together and written to a temporary file first, so a failed or cancelled run leaves your previous output untouched instead of a half-written file in its place
+- **Index builds no longer grow in memory with the size of the dump**: every batch of extracted text was kept until the build finished
+- **A background job that fails to start is now reported as failed**: it stayed `running` indefinitely and blocked every later job until the server was restarted
+- **The gem no longer ships maintainer scripts or images**. If you were running `scripts/fetch_*.rb` from an installed gem, take them from the repository instead
+- **The Docker Hub repository has been removed**: images are published to GitHub Container Registry only. If you still pull from Docker Hub, switch with `docker pull ghcr.io/yohasebe/wp2txt`
+
 ## [2.3.2] - 2026-08-13
 
 - **Documentation release — no code changes.** The changelog and guides bundled with the gem and the container image are rewritten to say what each change means for someone using wp2txt, rather than how it was implemented. The guide formerly at `docs/RESEARCH.md` is now [docs/INDEXES.md](docs/INDEXES.md) ("Offline Indexes, Queries, and the MCP Server")

data/DEVELOPMENT.md

@@ -400,13 +400,21 @@ docs/INDEXES.md is checked against the actual server surface by spec/docs_sync_s
 
 ## Docker
 
-Build and push Docker images:
+Images are published by GitHub Actions when a `v*` tag is pushed
+(`.github/workflows/publish-image.yml`), not from a maintainer's machine: a
+local build sends the working tree as its context, so untracked files ride
+along, while a runner starts from a clean checkout.
 
 ```bash
-rake check_image  # Builds the image locally and verifies it carries no private files
-rake push        # Verifies, then builds multi-arch and pushes to GHCR
+rake check_image   # Build locally and run the same gate the workflow runs
 ```
 
+The gate (`scripts/verify_image.rb`) lists what the image holds under `/wp2txt`
+and compares it against what Docker sends as the build context, in both
+directions. Anything unaccounted for fails, so a file the build starts
+producing must be added to `BUILD_ARTIFACTS` deliberately. To rehearse a
+publish, run the workflow from the Actions tab with **push** left off.
+
 ## Release Process
 
 1. Update version in `lib/wp2txt/version.rb`
@@ -414,7 +422,7 @@ rake push        # Verifies, then builds multi-arch and pushes to GHCR
 3. Run full test suite: `bundle exec rspec`
 4. Build gem: `gem build wp2txt.gemspec`
 5. Push to RubyGems: `gem push wp2txt-*.gem`
-6. Push Docker image: `rake push`
+6. Push the `v*` tag — GitHub Actions builds, gates, and publishes the image
 7. Create GitHub release
 
 ## Useful Links

data/DEVELOPMENT_ja.md

@@ -402,8 +402,7 @@ docs/INDEXES.md の MCP ツール表は spec/docs_sync_spec.rb が実際のサ
 Dockerイメージのビルドとプッシュ：
 
 ```bash
-rake check_image  # ローカルでイメージをビルドし、私的ファイルの混入がないか検証
-rake push        # 検証したうえでマルチアーキテクチャでビルドしGHCRにプッシュ
+rake check_image   # ローカルでビルドし、CI と同じゲートを走らせる
 ```
 
 ## リリースプロセス
@@ -413,7 +412,7 @@ rake push        # 検証したうえでマルチアーキテクチャでビル
 3. フルテストスイートを実行: `bundle exec rspec`
 4. gemをビルド: `gem build wp2txt.gemspec`
 5. RubyGemsにプッシュ: `gem push wp2txt-*.gem`
-6. Dockerイメージをプッシュ: `rake push`
+6. `v*` タグをプッシュ — GitHub Actions がビルド・検証・公開を行う
 7. GitHubリリースを作成
 
 ## 便利なリンク

data/README.md

@@ -84,7 +84,7 @@ docker run -it -v /path/to/localdata:/data ghcr.io/yohasebe/wp2txt
 
 The `wp2txt` command is available inside the container. Use `/data` for input/output files.
 
-Images are published to GitHub Container Registry (`ghcr.io/yohasebe/wp2txt`). The former Docker Hub repository (`yohasebe/wp2txt`) is no longer updated as of 2.3.1.
+Images are published to GitHub Container Registry (`ghcr.io/yohasebe/wp2txt`). The former Docker Hub repository has been removed — pull from GHCR.
 
 **MCP server (no Ruby required on the host):**
 

data/README_ja.md

@@ -80,7 +80,7 @@ docker run -it -v /path/to/localdata:/data ghcr.io/yohasebe/wp2txt
 
 `wp2txt`コマンドはコンテナ内で使用可能です。入出力には`/data`ディレクトリを使用してください。
 
-イメージはGitHub Container Registry（`ghcr.io/yohasebe/wp2txt`）で公開されています。旧 Docker Hub リポジトリ（`yohasebe/wp2txt`）は 2.3.1 以降更新されません。
+イメージはGitHub Container Registry（`ghcr.io/yohasebe/wp2txt`）で公開されています。旧 Docker Hub リポジトリは削除済みです — GHCR から pull してください。
 
 ## 基本的な使い方
 

data/Rakefile

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "bundler/gem_tasks"
+require "open3"
 require "rspec/core"
 require "rspec/core/rake_task"
 require_relative "./lib/wp2txt/version"
@@ -28,6 +29,10 @@ end
 
 Rake::Task["build"].enhance([:normalize_permissions])
 
+# Pre-release gate: verify the built gem's payload against spec.files and scan
+# it for names, content, and modes that must never ship (code-security protocol).
+Rake::Task["build"].enhance { sh "ruby", "scripts/verify_gem.rb" }
+
 # =============================================================================
 # Docker
 # =============================================================================
@@ -35,13 +40,14 @@ Rake::Task["build"].enhance([:normalize_permissions])
 # Paths that must never reach a published image. The image is built from the
 # working tree, so anything ignored locally (private notes, scratch files)
 # would otherwise ride along.
-IMAGE_FORBIDDEN_PATHS = %w[/wp2txt/research-notes /wp2txt/tmp /wp2txt/.git /wp2txt/CLAUDE.md /wp2txt/.claude].freeze
+IMAGE_FORBIDDEN_PATHS = %w[/wp2txt/research-notes /wp2txt/tmp /wp2txt/.git /wp2txt/CLAUDE.md /wp2txt/.claude /wp2txt/.private-doc-tokens].freeze
 
 desc "Verify a built image contains no private material (run before pushing)"
 task :verify_image, [:tag] do |_t, args|
   tag = args[:tag] || "wp2txt-verify:local"
   checks = IMAGE_FORBIDDEN_PATHS.map { |p| "test -e #{p} && echo LEAK:#{p}" }.join("; ")
-  out = `docker run --rm #{tag} sh -c '#{checks}; true' 2>&1`
+  out, status = Open3.capture2e("docker", "run", "--rm", tag, "sh", "-c", "#{checks}; true")
+  abort "Image verification failed for #{tag}: #{out}" unless status.success?
   leaks = out.lines.grep(/^LEAK:/).map(&:strip)
   abort "Image #{tag} contains private paths:\n  #{leaks.join("\n  ")}" unless leaks.empty?
 
@@ -54,18 +60,18 @@ task :check_image do
   Rake::Task[:verify_image].invoke
 end
 
-desc "Build and push Docker images to GHCR (verifies a local build first)"
-task push: :check_image do
-  # Docker Hub was retired after 2.3.0; GHCR (ghcr.io/yohasebe/wp2txt) is the
-  # only registry. Requires `docker buildx use multiarch` and a ghcr.io login.
-  sh <<-SCRIPT.strip_heredoc, { verbose: false }
-    /bin/bash -xeu <<'BASH'
-      # docker buildx create --name multiarch
-      # docker buildx use multiarch
-      # docker buildx inspect --bootstrap
-      docker buildx build --platform linux/amd64,linux/arm64 \
-        -t ghcr.io/yohasebe/wp2txt:#{Wp2txt::VERSION} -t ghcr.io/yohasebe/wp2txt:latest \
-        . --push
-    BASH
-  SCRIPT
+desc "Explain how images are published (they are built and pushed by CI)"
+task :push do
+  abort <<~MESSAGE
+    Images are published by GitHub Actions, not from here.
+
+    A local build sends this working tree as the build context, so untracked
+    files ride along; a runner starts from a clean checkout, where they do not
+    exist. Push a v* tag and .github/workflows/publish-image.yml takes over:
+
+        rake release            # tags and pushes (also publishes the gem)
+
+    To rehearse without publishing, run the workflow from the Actions tab with
+    "push" left off. To check a local build, run `rake check_image`.
+  MESSAGE
 end

data/bin/wp2txt

@@ -217,7 +217,8 @@ class WpApp
           next unless title_match
 
           title = title_match[1]
-          next if title.nil? || title.empty? || title.include?(":")
+          next if title.nil? || title.empty?
+          next unless Wp2txt.namespace_id(page_xml[%r{<ns>(-?\d+)</ns>}, 1]).zero?
 
           # Extract text content
           text_match = TEXT_REGEX.match(page_xml)
@@ -609,7 +610,8 @@ class WpApp
         next unless title_match
 
         title = title_match[1]
-        next if title.nil? || title.empty? || title.include?(":")
+        next if title.nil? || title.empty?
+        next unless Wp2txt.namespace_id(page_xml[%r{<ns>(-?\d+)</ns>}, 1]).zero?
 
         text_match = TEXT_REGEX.match(page_xml)
         next unless text_match

data/bin/wp2txt-mcp

@@ -73,7 +73,7 @@ def respond(&block)
   result = block.call
   MCP::Tool::Response.new([{ type: "text", text: JSON.generate(result) }])
 rescue ArgumentError => e
-  MCP::Tool::Response.new([{ type: "text", text: JSON.generate({ error: e.message }) }], error: true)
+  MCP::Tool::Response.new([{ type: "text", text: JSON.generate({ error: e.message, code: (e.code if e.respond_to?(:code)) }.compact) }], error: true)
 rescue StandardError => e
   MCP::Tool::Response.new([{ type: "text", text: JSON.generate({ error: "#{e.class}: #{e.message}" }) }], error: true)
 end
@@ -312,7 +312,8 @@ def extract_params(output_path:, content:, titles: nil, sections: nil, alias_set
     content: content, titles: titles,
     sections: sections, alias_set: alias_set, category: category, depth: depth,
     categories: categories, category_match: category_match,
-    title_match: title_match, limit: limit, chunk_size: chunk_size, chunk_overlap: chunk_overlap }
+    title_match: title_match, limit: limit, chunk_size: chunk_size, chunk_overlap: chunk_overlap,
+    overwrite: overwrite }
 end
 
 server.define_tool(
@@ -341,7 +342,7 @@ end
 
 server.define_tool(
   name: "cancel_job",
-  description: "Request cancellation of a running job (takes effect at the next batch boundary; the partial output file remains on disk).",
+  description: "Request cancellation of a running job (takes effect at the next batch boundary; staged output is removed).",
   input_schema: { properties: { job_id: { type: "string" } }, required: ["job_id"] }
 ) do |job_id:, server_context:|
   respond { job_manager.cancel(job_id) || { error: "unknown job: #{job_id}" } }

data/docs/INDEXES.md

@@ -130,6 +130,10 @@ $ claude mcp add wp2txt -- docker run -i --rm -v wp2txt:/root/.wp2txt ghcr.io/yo
   seconds, saved alias sets are re-checked before being stored, and files can only be
   written under the server's output directory — worth knowing if you plan to let an
   assistant work unattended.
+  Output confinement assumes a dedicated, trusted output directory: symlink paths are
+  rejected, both output and sidecar are exclusively reserved unless overwrite is requested,
+  and unique temporary files are renamed on success; concurrent replacement of parent
+  directories is outside this guarantee.
 
 ## 5. Cross-language SQL
 

data/lib/wp2txt/constants.rb

@@ -1,6 +1,11 @@
 # frozen_string_literal: true
 
 module Wp2txt
+  # Missing namespace elements use the metadata scanner's historical ns=0 default.
+  def self.namespace_id(value)
+    (value || "0").to_i
+  end
+
   # =========================================================================
   # Custom Exception Classes
   # =========================================================================