wp2txt 2.3.3 → 2.4.0

data/lib/wp2txt/stream_processor.rb

@@ -19,8 +19,12 @@ module Wp2txt
 
     attr_reader :buffer_size, :pages_processed, :bytes_read, :redirects_skipped
 
-    def initialize(input_path, bz2_gem: false, adaptive_buffer: true, validate_bz2: true, skip_redirects: true)
+    # keep_raw_text: also hand back each article's wikitext as stored in the
+    # dump (before comment removal), as :raw_text among the with_ids values
+    def initialize(input_path, bz2_gem: false, adaptive_buffer: true, validate_bz2: true, skip_redirects: true,
+                   keep_raw_text: false)
       @input_path = input_path
+      @keep_raw_text = keep_raw_text
       @bz2_gem = bz2_gem
       @buffer = +""
       @pending_bytes = +"".b
@@ -64,20 +68,28 @@ module Wp2txt
 
     # Iterate over each page in the input
     # Yields [title, text] for each page
-    def each_page
-      return enum_for(:each_page) unless block_given?
+    # Yields title and text of each article. With with_ids: true, also yields
+    # { page_id:, revision_id: } taken from the dump.
+    def each_page(with_ids: false, &block)
+      return enum_for(:each_page, with_ids: with_ids) unless block
+
+      emit = if with_ids
+               ->(title, text, ids) { block.call(title, text, ids) }
+             else
+               ->(title, text, _ids) { block.call(title, text) }
+             end
 
       if File.directory?(@input_path)
         # Process XML files in directory
         Dir.glob(File.join(@input_path, "*.xml")).sort.each do |xml_file|
-          process_xml_file(xml_file) { |title, text| yield title, text }
+          process_xml_file(xml_file) { |title, text, ids| emit.call(title, text, ids) }
         end
       elsif @input_path.end_with?(".bz2")
         # Process bz2 compressed file with streaming
-        process_bz2_stream { |title, text| yield title, text }
+        process_bz2_stream { |title, text, ids| emit.call(title, text, ids) }
       elsif @input_path.end_with?(".xml")
         # Process single XML file
-        process_xml_file(@input_path) { |title, text| yield title, text }
+        process_xml_file(@input_path) { |title, text, ids| emit.call(title, text, ids) }
       else
         raise ArgumentError, "Unsupported input format: #{@input_path}"
       end
@@ -162,20 +174,32 @@ module Wp2txt
     def fill_buffer
       chunk = @file_pointer.read(@buffer_size)
       unless chunk
-        @buffer << @pending_bytes.to_s.dup.force_encoding(Encoding::UTF_8).scrub("")
-        @pending_bytes = +"".b
+        unless @pending_bytes.to_s.empty?
+          raise Wp2txt::EncodingError,
+                "input ends in the middle of a UTF-8 character (byte #{@bytes_read}): #{@input_path}"
+        end
         return false
       end
 
-      @bytes_read += chunk.bytesize
-      bytes = @pending_bytes.to_s.b + chunk.b
-      # Retain a trailing UTF-8 sequence until the next read. Only complete
-      # chunks are scrubbed, so valid characters split by read are preserved.
+      carried = @pending_bytes.to_s.b
+      start = @bytes_read - carried.bytesize # stream position of the first carried byte
+      bytes = carried + chunk.b
+      # A read can end partway through a multi-byte character; carry those
+      # bytes into the next read rather than judging half a character.
       tail = bytes[/[\xC2-\xF4][\x80-\xBF]{0,2}\z/n]
       width = tail && (tail.getbyte(0) < 0xE0 ? 2 : tail.getbyte(0) < 0xF0 ? 3 : 4)
       @pending_bytes = tail && tail.bytesize < width ? tail : +"".b
-      bytes = bytes.byteslice(0, bytes.bytesize - @pending_bytes.bytesize)
-      @buffer << bytes.force_encoding(Encoding::UTF_8).scrub("")
+      bytes = bytes.byteslice(0, bytes.bytesize - @pending_bytes.bytesize).force_encoding(Encoding::UTF_8)
+      # Anything still invalid is corrupt input. Dropping it would change the
+      # text without a trace, so stop and say where it is.
+      unless bytes.valid_encoding?
+        bad = bytes.each_char.find_index { |c| !c.valid_encoding? }.to_i
+        offset = start + bytes[0, bad].bytesize
+        raise Wp2txt::EncodingError,
+              "invalid UTF-8 near decompressed byte #{offset}: #{@input_path}"
+      end
+      @bytes_read += chunk.bytesize
+      @buffer << bytes
 
       # Adaptive buffer adjustment: if memory is low, reduce buffer size
       if @adaptive_buffer && MemoryMonitor.memory_low?
@@ -237,6 +261,7 @@ module Wp2txt
       return nil unless Wp2txt.namespace_id(namespace).zero?
 
       text = text_node.content
+      raw_text = text if @keep_raw_text
 
       # Early redirect detection and skip (before expensive processing)
       # Redirects start with # or ＃ followed by redirect keyword and [[target]]
@@ -252,7 +277,9 @@ module Wp2txt
       end
 
       @pages_processed += 1
-      [title, text]
+      meta = Wp2txt.page_ids(page_xml)
+      meta[:raw_text] = raw_text if @keep_raw_text
+      [title, text, meta]
     rescue Nokogiri::XML::SyntaxError
       # Skip malformed XML
       nil

data/lib/wp2txt/template_expander.rb

@@ -171,6 +171,19 @@ module Wp2txt
       nil
     end
 
+    # Templates whose rendering the text-cleaning stage defines (see
+    # Wp2txt#correct_inline_template), keyed by normalized name. Limited to
+    # kinds whose output does not depend on marker settings.
+    def rendered_by_cleaner?(template_name)
+      @rendered_by_cleaner ||= [Wp2txt::RUBY_TEXT_TEMPLATES, Wp2txt::INTERWIKI_LINK_TEMPLATES]
+                               .flatten.to_set { |name| name.to_s.tr("_", " ").strip.downcase }
+      @rendered_by_cleaner.include?(template_name.tr("_", " "))
+    end
+
+    def text_renderer
+      @text_renderer ||= Object.new.extend(Wp2txt)
+    end
+
     def expand_single_template(content)
       parts = split_template_parts(content)
       return "" if parts.empty?
@@ -263,6 +276,12 @@ module Wp2txt
         # Handle lang-xx templates (e.g., lang-fr, lang-de, lang-ja)
         if template_name.start_with?("lang-")
           expand_lang_xx(template_name, params)
+        elsif rendered_by_cleaner?(template_name)
+          # These carry words the article needs (a reading, a link's display
+          # text). Deleting them dropped those words; leaving the raw template
+          # for later confused links that contain it (an image caption with a
+          # "|" inside). Render them now, with the cleaner's own rules.
+          text_renderer.correct_inline_template("{{#{expand(content)}}}")
         else
           @preserve_unknown ? "{{#{content}}}" : ""
         end

data/lib/wp2txt/utils.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "strscan"
+require_relative "wikitext_regions"
 require_relative "constants"
 require_relative "regex"
 require_relative "text_processing"
@@ -574,8 +575,9 @@ module Wp2txt
   # Helper to check if template name matches any in a list (case-insensitive)
   def template_matches?(name, template_list)
     return false if template_list.nil? || template_list.empty?
-    normalized_name = name.to_s.strip.downcase
-    template_list.any? { |t| t.downcase == normalized_name }
+    # MediaWiki treats "_" and " " in template names as the same character
+    normalized_name = name.to_s.tr("_", " ").strip.downcase
+    template_list.any? { |t| t.tr("_", " ").downcase == normalized_name }
   end
 
   def correct_inline_template(str, enabled_markers = [], extract_citations = false)
@@ -583,7 +585,7 @@ module Wp2txt
     return str unless str.include?("{{")
 
     process_nested_single_pass(str, "{{", "}}") do |contents|
-      parts = contents.split("|")
+      parts = WikitextRegions.split_pipes(contents)
       template_name = (parts[0] || "").strip
       template_name_lower = template_name.downcase
 

data/lib/wp2txt/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Wp2txt
-  VERSION = "2.3.3"
+  VERSION = "2.4.0"
 end

data/lib/wp2txt/wikitext_regions.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module Wp2txt
+  # Literal regions suppress wikitext parsing. Galleries and timelines can
+  # contain links, but their contents are not the article's lead prose.
+  module WikitextRegions
+    LITERAL_TAGS = %w[nowiki pre math chem ce score syntaxhighlight source graph mapframe templatedata].freeze
+    NON_PROSE_TAGS = %w[gallery timeline].freeze
+    COMMENT = /<!--.*?(?:-->|\z)/m
+
+    def self.region_pattern(tags)
+      /<!--.*?(?:-->|\z)|<(?<tag>#{tags.join('|')})(?=[\s\/>])(?:"[^"]*"|'[^']*'|[^'">])*?(?:\/>|>.*?(?:<\/\k<tag>\s*>|\z))/mi
+    end
+
+    LITERAL_REGION = region_pattern(LITERAL_TAGS)
+    LEAD_REGION = region_pattern(LITERAL_TAGS + NON_PROSE_TAGS)
+    LITERAL_REGION_AT = /\G(?:#{LITERAL_REGION})/
+    LEAD_REGION_AT = /\G(?:#{LEAD_REGION})/
+
+    module_function
+
+    def end_at(text, offset, lead: false)
+      match = (lead ? LEAD_REGION_AT : LITERAL_REGION_AT).match(text, offset)
+      match.end(0) if match
+    end
+
+    def remove_literal(text)
+      text.gsub(LITERAL_REGION, "")
+    end
+
+    # One space per codepoint, including internal newlines: an excluded
+    # region must not introduce a paragraph or heading boundary.
+    def mask_lead(text)
+      text.gsub(LEAD_REGION) { |region| " " * region.length }
+    end
+
+    def split_pipes(text)
+      parts = [+""]
+      stack = []
+      i = 0
+      while i < text.length
+        two = text[i, 2]
+        if text[i] == "<" && (stop = end_at(text, i))
+          parts.last << text[i...stop]
+          i = stop
+        elsif ["{{", "[["].include?(two)
+          stack << (two == "{{" ? "}}" : "]]")
+          parts.last << two
+          i += 2
+        elsif !stack.empty? && two == stack.last
+          stack.pop
+          parts.last << two
+          i += 2
+        else
+          if text[i] == "|" && stack.empty?
+            parts << +""
+          else
+            parts.last << text[i]
+          end
+          i += 1
+        end
+      end
+      parts
+    end
+  end
+end

data/lib/wp2txt.rb

@@ -269,12 +269,14 @@ module Wp2txt
         end
         page << line if inside_page
       end
-      if page.empty?
-        false
-      else
-        page.force_encoding("utf-8")
+      return false if page.empty?
+
+      page.force_encoding("utf-8")
+      # Corrupt input must not be cleaned into different text without a trace
+      unless page.valid_encoding?
+        title = page[%r{<title>([^<]*)</title>}, 1]&.scrub("?")
+        raise Wp2txt::EncodingError, "invalid UTF-8 in page #{title.inspect} of #{@input_file}"
       end
-    rescue ::Encoding::InvalidByteSequenceError, ::Encoding::UndefinedConversionError
       page
     end
 

data/spec/docs_sync_spec.rb

@@ -37,14 +37,14 @@ RSpec.describe "documentation surface sync" do
     expect(phantom).to be_empty, "documented tools that do not exist: #{phantom.join(', ')}"
   end
 
-  # Tripwire: tracked files must not contain tokens listed in .private-doc-tokens,
+  # Tripwire: tracked files must not contain tokens listed in .local/doc-tokens,
   # an untracked, machine-local file (one substring per line; # starts a comment).
   # The file exists only on machines that maintain such a list; everywhere else
   # (CI, other contributors) this example skips — loudly, so a silently dead
   # check cannot be mistaken for a passing one.
   it "keeps machine-local private tokens out of tracked files" do
-    token_file = File.join(repo_root, ".private-doc-tokens")
-    skip "SKIPPED: no .private-doc-tokens on this machine — tripwire not checked" unless File.exist?(token_file)
+    token_file = File.join(repo_root, ".local", "doc-tokens")
+    skip "SKIPPED: no .local/doc-tokens on this machine — tripwire not checked" unless File.exist?(token_file)
 
     tokens = File.readlines(token_file, encoding: "UTF-8")
                  .map(&:strip).reject { |t| t.empty? || t.start_with?("#") }

data/spec/langlinks_importer_spec.rb

@@ -65,6 +65,33 @@ RSpec.describe Wp2txt::LanglinksImporter do
     rows
   end
 
+  # Current dumps put the INSERT header on its own line and one tuple per line
+  MULTILINE_SQL = <<~MSQL
+    INSERT INTO `langlinks` VALUES
+    (1,'en','Film A'),
+    (1,'ja','映画A'),
+    (2,'en','It\\'s a Film; Really');
+    INSERT INTO `pagelinks` VALUES
+    (9,'Ignored',0);
+    INSERT INTO `langlinks` VALUES
+    (3,'fr','Film B');
+  MSQL
+
+  describe "dumps with one tuple per line" do
+    it "imports every tuple of every langlinks statement, and nothing else" do
+      path = write_langlinks("testwiki-20260101-langlinks.sql.gz", MULTILINE_SQL, gzip: true)
+      expect(import(path)[:row_count]).to eq(4)
+      expect(langlinks_rows).to contain_exactly(
+        [1, "en", "Film A"], [1, "ja", "映画A"], [2, "en", "It's a Film; Really"], [3, "fr", "Film B"]
+      )
+    end
+
+    it "refuses to report success when no rows could be read" do
+      path = write_langlinks("testwiki-20260101-langlinks.sql", "-- nothing here\nUNLOCK TABLES;\n")
+      expect { import(path) }.to raise_error(Wp2txt::Error, /no langlinks rows found/)
+    end
+  end
+
   describe "parsing and normalization" do
     it "imports tuples with escapes, commas, parens, and multiple INSERT statements" do
       path = write_langlinks("testwiki-20260101-langlinks.sql")

data/spec/lead_terms_edge_cases_spec.rb

@@ -0,0 +1,194 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require 'wp2txt'
+require 'wp2txt/lead_terms'
+require 'wp2txt/link_counter'
+require 'wp2txt/metadata_index'
+require 'open3'
+require 'tmpdir'
+require_relative 'support/multistream_fixture'
+
+RSpec.describe 'lead terms and link counting on unusual wikitext' do
+  include MultistreamFixture
+
+  let(:cleaner) { Object.new.extend(Wp2txt) }
+  let(:render) { ->(s) { cleaner.format_wiki(s, expand_templates: true, markers: [:all]) } }
+  def terms(text)
+    Wp2txt::LeadTerms.extract(text, render: render)
+  end
+
+  it '#1 preserves link display and reading with expansion enabled or disabled' do
+    {
+      '{{読み仮名|[[東京|東京都]]|とうきょう}}' => '東京都（とうきょう）',
+      '{{仮リンク|[[東京|東京都]]|en|Tokyo}}' => '東京都',
+      '{{読み仮名|{{lang|ja|[[東京|東京都]]}}|とうきょう}}' => '東京都（とうきょう）',
+      '{{読み仮名|東京|}}' => '東京'
+    }.each do |input, expected|
+      [true, false].each do |expand|
+        expect(cleaner.format_wiki(input, expand_templates: expand)).to eq(expected)
+      end
+    end
+  end
+
+  def build_case_dump(dir, rule)
+    alias_title = rule == 'first-letter' ? 'Alias' : 'alias'
+    pages = [[1, 'apple', ''], [2, 'Apple', ''], [3, alias_title, '#REDIRECT [[apple]]'],
+             [4, 'Direct', '[[apple]]'], [5, 'Via', '[[alias]]'], [6, '東京', '']]
+    xml = "<mediawiki><siteinfo><case>#{rule}</case></siteinfo>" + pages.map do |id, title, text|
+      page_xml(id: id, ns: 0, title: title, text: text)
+    end.join + '</mediawiki>'
+    dump = File.join(dir, 'testwiki-20260101-multistream.xml.bz2')
+    File.binwrite(dump, bzip2(xml))
+    db = File.join(dir, 'meta.sqlite3')
+    Wp2txt::MetadataIndexBuilder.new(dump, [0], db_path: db, num_processes: 0).build.close
+    [dump, db]
+  end
+
+  it '#2 stores siteinfo case and preserves direct and redirected case-sensitive targets' do
+    Dir.mktmpdir do |dir|
+      dump, path = build_case_dump(dir, 'case-sensitive')
+      db = SQLite3::Database.new(path)
+      expect(db.get_first_value("SELECT value FROM metadata WHERE key = 'case_rule'")).to eq('case-sensitive')
+      db.close
+      Wp2txt::LinkCounter.new(dump, [0], db_path: path, num_processes: 0).count!
+      db = SQLite3::Database.new(path)
+      expect(db.execute('SELECT page_id,inlinks,via_redirects FROM page_inlinks WHERE page_id IN (1,2) ORDER BY page_id'))
+        .to eq([[1, 2, 1], [2, 0, 0]])
+      db.close
+      expect(Wp2txt::MetadataIndex.normalize_title('apple')).to eq('Apple')
+    end
+  end
+
+  it '#2 uses first-letter for explicit siteinfo and legacy metadata without the rule' do
+    Dir.mktmpdir do |dir|
+      dump, path = build_case_dump(dir, 'first-letter')
+      [false, true].each do |legacy|
+        db = SQLite3::Database.new(path)
+        if legacy
+          db.execute("DELETE FROM metadata WHERE key = 'case_rule'")
+        else
+          expect(db.get_first_value("SELECT value FROM metadata WHERE key = 'case_rule'")).to eq('first-letter')
+        end
+        db.close
+        Wp2txt::LinkCounter.new(dump, [0], db_path: path, num_processes: 0).count!
+        db = SQLite3::Database.new(path)
+        expect(db.get_first_value('SELECT inlinks FROM page_inlinks WHERE page_id=1')).to eq(0)
+        expect(db.execute('SELECT inlinks,via_redirects FROM page_inlinks WHERE page_id=2')).to eq([[2, 1]])
+        db.close
+      end
+    end
+  end
+
+  it '#2 reads a separate siteinfo stream before the first indexed page' do
+    Dir.mktmpdir do |dir|
+      header = bzip2('<mediawiki><siteinfo><case>case-sensitive</case></siteinfo>')
+      dump = File.join(dir, 'dump.xml.bz2')
+      File.binwrite(dump, header + bzip2(page_xml(id: 1, ns: 0, title: 'apple', text: '') + '</mediawiki>'))
+      path = File.join(dir, 'meta.sqlite3')
+      Wp2txt::MetadataIndexBuilder.new(dump, [header.bytesize], db_path: path, num_processes: 0).build.close
+      db = SQLite3::Database.new(path)
+      expect(db.get_first_value("SELECT value FROM metadata WHERE key = 'case_rule'")).to eq('case-sensitive')
+      db.close
+    end
+  end
+
+  %w[nowiki pre math syntaxhighlight source gallery score timeline chem ce graph mapframe templatedata].each do |tag|
+    it "#3 skips #{tag} contents and self-closing forms while preserving source offsets" do
+      text = "<#{tag.upcase} data-x='>'>'''偽'''（にせ）{{ruby|偽|にせ}}</#{tag.upcase}>\n\n" \
+             "<#{tag} />'''東京'''（とうきょう）。"
+      result = terms(text)
+      expect(result.map { |t| t['text'] }).to eq(['東京'])
+      s, e = result.first['span']['bold']
+      expect(text[s...e]).to eq("'''東京'''")
+      expect(terms("<#{tag}>'''偽'''" )).to eq([])
+    end
+  end
+
+  it '#4 ignores delimiters in comments and excluded tags inside parentheses' do
+    text = "'''東京'''（とうきょう<!-- ) , -->、<nowiki>),</nowiki>Tokyo）は都市。"
+    result = terms(text).first
+    s, e = result['span']['paren']
+    expect(text[s...e]).to eq('（とうきょう<!-- ) , -->、<nowiki>),</nowiki>Tokyo）')
+    expect(result['notes'].size).to eq(2)
+    expect(result['notes'].first).to eq('とうきょう')
+    expect(result['notes_text']).not_to include('<!--')
+  end
+
+  it 'retains the original text of ruby arguments containing excluded regions' do
+    expect(terms('{{ruby|<nowiki>東京</nowiki>|とうきょう}}').first.slice('text', 'reading'))
+      .to eq('text' => '東京', 'reading' => 'とうきょう')
+  end
+
+  it '#5 detects only headings outside skipped regions including trailing comments' do
+    ["<!--\n== 偽 ==\n-->", "<nowiki>\n== 偽 ==\n</nowiki>", "{{box|\n== 偽 ==\n}}", "{|\n== 偽 ==\n|}"].each do |prefix|
+      expect(terms("#{prefix}\n'''東京'''（とうきょう）").map { |t| t['text'] }).to eq(['東京'])
+    end
+    expect(terms("導入文。\n\n== 歴史 == <!-- comment -->\n'''後の語'''（あと）。")).to eq([])
+  end
+
+  it '#6 allows single newlines before and inside notes but stops at a blank line' do
+    expect(terms("'''東京'''\n（とうきょう）は都市。").first['notes']).to eq(['とうきょう'])
+    text = "'''東京'''（とうきょう、\nTokyo）は都市。"
+    term = terms(text).first
+    expect(term['notes']).to eq(['とうきょう', 'Tokyo'])
+    s, e = term['span']['paren']
+    expect(text[s...e]).to eq("（とうきょう、\nTokyo）")
+    ["'''東京'''\n\n（とうきょう）", "'''東京'''（とうきょう、\n \nTokyo）"].each do |input|
+      expect(terms(input).first['notes']).to eq([])
+    end
+  end
+
+  it '#7 stops at empty lines containing whitespace' do
+    ["\n \n", "\n\t\n", "\r\n \r\n"].each do |gap|
+      expect(terms("'''東京'''（とうきょう）。#{gap}'''別段落'''（べつだんらく）。").map { |t| t['text'] }).to eq(['東京'])
+    end
+  end
+
+  def counts(text)
+    counter = Wp2txt::LinkCounter.new('', [], db_path: '')
+    counter.instance_variable_set(:@redirects, {})
+    direct, via = Hash.new(0), Hash.new(0)
+    counter.send(:count_page, page_xml(id: 1, ns: 0, title: 'Source', text: text), direct, via)
+    direct
+  end
+
+  it '#8 decodes title entities before sections and normalizes whitespace before colon' do
+    expect(counts('[[M&amp;A]] [[Caf&#233;]] [[ :東京#節|表示]] [[Caf&#xE9;#section]] [[#節]] [[]]'))
+      .to eq('M&A' => 1, 'Café' => 1, '東京' => 1)
+    expect(counts('[[東京&#35;節]] [[東京|表示]]')).to eq('東京' => 1)
+  end
+
+  it '#9 excludes every non-wikitext tag and comments from incoming links' do
+    %w[nowiki pre math syntaxhighlight source score chem ce graph mapframe templatedata].each do |tag|
+      expect(counts("<#{tag}>[[東京]]</#{tag}>[[大阪]]<#{tag}/><!-- [[京都]] -->")).to eq('大阪' => 1)
+    end
+  end
+
+  it 'handles empty text and binary invalid bytes in the shared region helper' do
+    expect(terms('')).to eq([])
+    expect(counts('')).to eq({})
+    expect(Wp2txt::WikitextRegions.remove_literal("<nowiki>\xFF</nowiki>\xFE".b)).to eq("\xFE".b)
+  end
+
+  def cli(*options)
+    Dir.mktmpdir do |dir|
+      input = File.join(dir, 'x.xml.bz2')
+      File.binwrite(input, bzip2("<mediawiki>#{page_xml(id: 1, ns: 0, title: '東京', text: "'''東京'''。")}</mediawiki>"))
+      Open3.capture3(RbConfig.ruby, '-I', File.expand_path('../lib', __dir__),
+                     File.expand_path('../bin/wp2txt', __dir__), '-i', input, '-o', dir, *options)
+    end
+  end
+
+  it '#10 rejects lead terms with ractor' do
+    _, err, status = cli('--format', 'json', '--lead-terms', '--ractor', '--no-turbo')
+    expect(status.success?).to be(false)
+    expect(err).to include('--lead-terms cannot be combined with --ractor')
+  end
+
+  it '#10 warns about missing source identifiers in ractor JSON without rejecting it' do
+    _, err, status = cli('--format', 'json', '--ractor', '--no-turbo', '-n', '1')
+    expect(status.success?).to be(true), err
+    expect(err).to include('does not include page IDs or revision IDs')
+  end
+end