wp2txt 2.3.1 → 2.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.dockerignore +3 -0
- data/.github/workflows/publish-image.yml +140 -0
- data/.gitignore +1 -0
- data/CHANGELOG.md +92 -281
- data/DEVELOPMENT.md +20 -4
- data/DEVELOPMENT_ja.md +9 -3
- data/README.md +13 -14
- data/README_ja.md +8 -8
- data/Rakefile +23 -17
- data/bin/wp2txt +4 -2
- data/bin/wp2txt-mcp +4 -3
- data/docs/{RESEARCH.md → INDEXES.md} +69 -64
- data/lib/wp2txt/constants.rb +5 -0
- data/lib/wp2txt/corpus.rb +96 -79
- data/lib/wp2txt/corpus_jobs.rb +15 -13
- data/lib/wp2txt/fts_index.rb +12 -3
- data/lib/wp2txt/metadata_index.rb +3 -3
- data/lib/wp2txt/output_path.rb +72 -12
- data/lib/wp2txt/stream_processor.rb +19 -9
- data/lib/wp2txt/version.rb +1 -1
- data/spec/docs_sync_spec.rb +27 -5
- data/spec/fts_index_spec.rb +40 -0
- data/spec/p1_correctness_spec.rb +357 -0
- data/spec/stream_processor_spec.rb +3 -1
- data/spec/titles_output_path_spec.rb +49 -1
- data/wp2txt.gemspec +1 -1
- metadata +5 -10
- data/image/wp2txt-logo.svg +0 -16
- data/image/wp2txt.svg +0 -31
- data/scripts/benchmark_regex.rb +0 -161
- data/scripts/fetch_html_entities.rb +0 -94
- data/scripts/fetch_language_metadata.rb +0 -180
- data/scripts/fetch_mediawiki_data.rb +0 -334
- data/scripts/fetch_template_data.rb +0 -186
- data/scripts/profile_memory.rb +0 -139
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 051b5501ecf2aad35ca972af6527c5afbce88222c17e9c93dc76f806b91a8e9d
|
|
4
|
+
data.tar.gz: 02efabb7102da40c144a3c8204f564a8a40dfa1e77a54813e40bd6740693a264
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: d89ff3e231200c36f647284fee43565021c69c73ac8d9f23ca42b58008d2d0d3ac5598fe63aa3736a45ce933622dd28f649f721635a63e782d638f444602d08b
|
|
7
|
+
data.tar.gz: c70fccdb332688f805e77dbefd291563045e2f53c07e7cc994dbff0a7f8858dded28f18bce4bcb1296e7594f12ee54eea12a4573beee04a8944d4e39b7d35620
|
data/.dockerignore
CHANGED
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
name: Publish image
|
|
2
|
+
|
|
3
|
+
# Publishes the multi-arch container image to the GitHub Container Registry.
|
|
4
|
+
#
|
|
5
|
+
# Publishing happens here rather than from a maintainer's machine on purpose. A
|
|
6
|
+
# local build sends the working tree as the build context, so anything sitting
|
|
7
|
+
# there untracked rides along; a runner starts from a clean checkout, where such
|
|
8
|
+
# files do not exist. Twice, private files reached published images this way.
|
|
9
|
+
#
|
|
10
|
+
# Fires on a v* tag. Run it by hand from the Actions tab for a dry run: the
|
|
11
|
+
# default leaves `push` off, so it builds and gates without publishing.
|
|
12
|
+
|
|
13
|
+
on:
|
|
14
|
+
push:
|
|
15
|
+
tags: ['v*']
|
|
16
|
+
workflow_dispatch:
|
|
17
|
+
inputs:
|
|
18
|
+
push:
|
|
19
|
+
description: 'Publish to GHCR (off = build and gate only)'
|
|
20
|
+
type: boolean
|
|
21
|
+
default: false
|
|
22
|
+
|
|
23
|
+
env:
|
|
24
|
+
IMAGE: ghcr.io/yohasebe/wp2txt
|
|
25
|
+
|
|
26
|
+
jobs:
|
|
27
|
+
publish:
|
|
28
|
+
name: Build, gate, publish
|
|
29
|
+
runs-on: ubuntu-latest
|
|
30
|
+
permissions:
|
|
31
|
+
contents: read
|
|
32
|
+
packages: write
|
|
33
|
+
|
|
34
|
+
steps:
|
|
35
|
+
- uses: actions/checkout@v4
|
|
36
|
+
|
|
37
|
+
- name: Work out the version and whether to publish
|
|
38
|
+
id: plan
|
|
39
|
+
run: |
|
|
40
|
+
set -euo pipefail
|
|
41
|
+
file_version=$(grep -oE '[0-9]+\.[0-9]+\.[0-9]+' lib/wp2txt/version.rb | head -1)
|
|
42
|
+
if [ -z "$file_version" ]; then
|
|
43
|
+
echo "could not read a version from lib/wp2txt/version.rb" >&2
|
|
44
|
+
exit 1
|
|
45
|
+
fi
|
|
46
|
+
|
|
47
|
+
if [ "${GITHUB_EVENT_NAME}" = "push" ]; then
|
|
48
|
+
tag_version="${GITHUB_REF_NAME#v}"
|
|
49
|
+
if [ "$tag_version" != "$file_version" ]; then
|
|
50
|
+
echo "tag ${GITHUB_REF_NAME} disagrees with lib/wp2txt/version.rb (${file_version})" >&2
|
|
51
|
+
echo "Publishing would tag the image with a version the code does not claim." >&2
|
|
52
|
+
exit 1
|
|
53
|
+
fi
|
|
54
|
+
publish=true
|
|
55
|
+
else
|
|
56
|
+
publish="${{ inputs.push }}"
|
|
57
|
+
fi
|
|
58
|
+
|
|
59
|
+
echo "version=$file_version" >> "$GITHUB_OUTPUT"
|
|
60
|
+
echo "publish=$publish" >> "$GITHUB_OUTPUT"
|
|
61
|
+
echo "Version $file_version; publish=$publish"
|
|
62
|
+
|
|
63
|
+
- uses: ruby/setup-ruby@v1
|
|
64
|
+
with:
|
|
65
|
+
ruby-version: '3.3'
|
|
66
|
+
|
|
67
|
+
- uses: docker/setup-qemu-action@v3
|
|
68
|
+
|
|
69
|
+
- uses: docker/setup-buildx-action@v3
|
|
70
|
+
|
|
71
|
+
# Single arch, loaded into the daemon, so the gate can open it. The
|
|
72
|
+
# multi-arch build below reuses these layers from the builder's cache.
|
|
73
|
+
- name: Build for inspection (linux/amd64)
|
|
74
|
+
uses: docker/build-push-action@v6
|
|
75
|
+
with:
|
|
76
|
+
context: .
|
|
77
|
+
platforms: linux/amd64
|
|
78
|
+
load: true
|
|
79
|
+
push: false
|
|
80
|
+
tags: ${{ env.IMAGE }}:gate-candidate
|
|
81
|
+
|
|
82
|
+
- name: Gate — compare the image against the build context
|
|
83
|
+
run: ruby scripts/verify_image.rb "${IMAGE}:gate-candidate"
|
|
84
|
+
|
|
85
|
+
- name: Log in to GHCR
|
|
86
|
+
if: steps.plan.outputs.publish == 'true'
|
|
87
|
+
uses: docker/login-action@v3
|
|
88
|
+
with:
|
|
89
|
+
registry: ghcr.io
|
|
90
|
+
username: ${{ github.actor }}
|
|
91
|
+
password: ${{ secrets.GITHUB_TOKEN }}
|
|
92
|
+
|
|
93
|
+
- name: Build and push (linux/amd64, linux/arm64)
|
|
94
|
+
id: publish
|
|
95
|
+
if: steps.plan.outputs.publish == 'true'
|
|
96
|
+
uses: docker/build-push-action@v6
|
|
97
|
+
with:
|
|
98
|
+
context: .
|
|
99
|
+
platforms: linux/amd64,linux/arm64
|
|
100
|
+
push: true
|
|
101
|
+
tags: |
|
|
102
|
+
${{ env.IMAGE }}:${{ steps.plan.outputs.version }}
|
|
103
|
+
${{ env.IMAGE }}:latest
|
|
104
|
+
labels: |
|
|
105
|
+
org.opencontainers.image.source=https://github.com/yohasebe/wp2txt
|
|
106
|
+
org.opencontainers.image.version=${{ steps.plan.outputs.version }}
|
|
107
|
+
|
|
108
|
+
# Reading it back without credentials is the only proof that a user can
|
|
109
|
+
# pull what was just published.
|
|
110
|
+
- name: Check the published digest is readable anonymously
|
|
111
|
+
if: steps.plan.outputs.publish == 'true'
|
|
112
|
+
run: |
|
|
113
|
+
set -euo pipefail
|
|
114
|
+
digest="${{ steps.publish.outputs.digest }}"
|
|
115
|
+
docker logout ghcr.io
|
|
116
|
+
for attempt in 1 2 3 4 5; do
|
|
117
|
+
if docker manifest inspect "${IMAGE}@${digest}" > /dev/null 2>&1; then
|
|
118
|
+
echo "readable anonymously: ${IMAGE}@${digest}"
|
|
119
|
+
exit 0
|
|
120
|
+
fi
|
|
121
|
+
echo "attempt ${attempt} failed; ghcr.io can lag a few seconds, retrying in 10s"
|
|
122
|
+
sleep 10
|
|
123
|
+
done
|
|
124
|
+
echo "${IMAGE}@${digest} is not readable anonymously" >&2
|
|
125
|
+
exit 1
|
|
126
|
+
|
|
127
|
+
- name: Summary
|
|
128
|
+
if: always()
|
|
129
|
+
run: |
|
|
130
|
+
{
|
|
131
|
+
echo "### wp2txt image"
|
|
132
|
+
echo
|
|
133
|
+
echo "- version: \`${{ steps.plan.outputs.version }}\`"
|
|
134
|
+
echo "- published: \`${{ steps.plan.outputs.publish }}\`"
|
|
135
|
+
if [ -n "${{ steps.publish.outputs.digest }}" ]; then
|
|
136
|
+
echo "- digest: \`${{ steps.publish.outputs.digest }}\`"
|
|
137
|
+
echo "- pull: \`docker pull ${IMAGE}:${{ steps.plan.outputs.version }}\`"
|
|
138
|
+
fi
|
|
139
|
+
echo "- run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
|
|
140
|
+
} >> "$GITHUB_STEP_SUMMARY"
|
data/.gitignore
CHANGED