vangrail 0.1.0 → 0.2.0

data/lib/vangrail/nlp.rb

@@ -0,0 +1,596 @@
+# frozen_string_literal: true
+
+require 'set'
+
+module Vangrail
+  # Text analysis in the standard library: normalisation, a suffix stripper, a
+  # concept lexicon with negation, and set similarity over character n-grams.
+  #
+  # A regexp matches the string an attacker wrote. Every published corpus shows
+  # the same instruction arriving in a hundred wordings, and the wordings cost
+  # an attacker one edit each while each new pattern costs a maintainer a
+  # false-positive budget. What survives rewording is not the string; it is the
+  # small set of concepts the sentence has to contain to do its job. An
+  # injection has to name an override, or a secret, or an audience to hide
+  # from, because a sentence that names none of them is not asking for
+  # anything.
+  #
+  # So the text is reduced to concepts before anything judges it. That is a
+  # lexicon, a stemmer, and a negation rule, which is decades-old NLP and runs
+  # in microseconds with nothing loaded from disk. It is not an embedding and
+  # cannot be: a synonym outside the lexicon is a miss, and the lexicon is
+  # visible in this file for exactly that reason.
+  #
+  # Concepts are language-independent and words are not, which is what makes a
+  # second language a word list rather than a rewrite. Dutch ships beside
+  # English and both are read by default, because at a Dutch institution the
+  # handbook, the wiki, and the page an attacker edits are as likely to be in
+  # Dutch as in English, and a rail that reads only English is a rail that
+  # reads only half the corpus it was pointed at.
+  module NLP
+    module_function
+
+    # UTF-8 or something that can be read as it. A body off a socket arrives
+    # tagged ASCII-8BIT whatever is in it, so the tag is corrected before
+    # anything else reads the bytes. Retag first, scrub second: the order
+    # matters, because scrub on ASCII-8BIT does nothing useful and
+    # unicode_normalize refuses a binary-tagged body whatever the bytes are.
+    def usable(text)
+      body = text.to_s
+      body = body.dup.force_encoding(Encoding::UTF_8) unless body.encoding == Encoding::UTF_8
+      body.valid_encoding? ? body : body.scrub
+    end
+
+    # Fold to a comparable form: NFKC so fullwidth and compatibility forms
+    # collapse onto ASCII, downcase, and every run of non-alphanumerics to a
+    # single space. Punctuation is separator rather than signal here, which is
+    # what makes "ignore-all-previous-instructions" and the spaced form the
+    # same token sequence, and what splits "API-sleutel" into the two words a
+    # lexicon can hold.
+    #
+    # Diacritics survive, because \p{Alnum} is not ASCII: "beëindig" is one
+    # token and stays one.
+    def normalize(text)
+      usable(text).unicode_normalize(:nfkc).downcase.gsub(/[^\p{Alnum}]+/, ' ').strip
+    end
+
+    def words(text)
+      normalize(text).split
+    end
+
+    # Stemming is the hot path: every rule reads every token of every clause,
+    # and real prose repeats its words. The memo turns six regexp attempts per
+    # token into a hash hit for everything after the first sighting.
+    #
+    # Deliberately mutable, and deliberately without a lock. Two threads racing
+    # here lose an entry and recompute it, which costs one string comparison
+    # and cannot produce a wrong answer, because the value is a pure function
+    # of the key.
+    STEM_LIMIT = 8192
+    STEM_CACHE = {} # rubocop:disable Style/MutableConstant
+
+    # An English suffix stripper, not Porter, and not applied per language.
+    #
+    # It exists so the lexicon can list one form of a word instead of five, and
+    # correctness is not the requirement: consistency is. Text and lexicon are
+    # stemmed by this same function, so a stem that is not a word ("hiding" to
+    # "hid") still matches, and a form the rules mangle is listed in the
+    # lexicon in the form it appears.
+    #
+    # Dutch takes the consequence of that squarely: its plural is -en, which
+    # these rules do not touch, and stripping it would maul English ("token" to
+    # "tok"). So the Dutch lexicon lists singular and plural separately. A word
+    # list is cheap; a stemmer that silently collides two languages' vocabulary
+    # is not.
+    def stem(word)
+      text = word.to_s
+      return text if text.length <= 3
+
+      return STEM_CACHE[text] if STEM_CACHE.key?(text)
+
+      stemmed = strip_suffix(text)
+      # Bounded because the input is hostile. A memo that keeps the first
+      # writers forever is a cache an attacker fills with unique tokens; once
+      # full, the oldest key is evicted so ordinary words can still land.
+      STEM_CACHE.shift if STEM_CACHE.size >= STEM_LIMIT
+      STEM_CACHE[text] = stemmed
+      stemmed
+    end
+
+    def strip_suffix(word)
+      case word
+      when /\A(.+)ies\z/ then "#{Regexp.last_match(1)}y"
+      when /\A(.+ss)es\z/ then Regexp.last_match(1)
+      when /\A(.+[^su])s\z/ then Regexp.last_match(1)
+      when /\A(.{3,})ing\z/ then Regexp.last_match(1)
+      when /\A(.{3,})ed\z/ then Regexp.last_match(1)
+      when /\A(.{3,})ly\z/ then Regexp.last_match(1)
+      else word
+      end
+    end
+
+    # The concepts an injection has to name, and the words that name them, per
+    # language.
+    #
+    # A word can carry more than one: "prompt" is an instruction when something
+    # overrides it and a secret when something asks for it, and which one it is
+    # depends on the other concept in the window rather than on the word. So
+    # the lexicon maps a word to every concept it can carry and lets the
+    # templates disambiguate.
+    #
+    # Written out rather than generated, because a lexicon that nobody can read
+    # is a rule nobody can audit, and every entry here is a phrase somebody has
+    # to be able to argue with.
+    #
+    # No bare word for a key is in either secret list. "Print your SSH key" and
+    # "toon je publieke sleutel" are handbook sentences, and the secret worth
+    # naming is the one an application holds, which the phrase table catches as
+    # "api key" and "api sleutel".
+    CONCEPTS = {
+      en: {
+        # Verbs that cancel something already said.
+        override: %w[ignore disregard forget override bypass discard dismiss overlook],
+        # What gets cancelled: the standing instruction, in any of its names.
+        instruction: %w[instruction direction directive rule guideline guidance order command prompt policy],
+        # A reference backwards, to whatever was said before this text.
+        prior: %w[previous prior above earlier preceding foregoing aforementioned original initial],
+        # Verbs that move text out: to the reader, to a log, to a URL.
+        reveal: %w[reveal print show display disclose repeat output echo dump send post transmit leak recite
+                   tell mention share admit acknowledge],
+        # What is worth moving out.
+        secret: %w[prompt token credential password secret configuration instruction],
+        # Verbs that keep something in.
+        conceal: %w[hide conceal omit withhold suppress],
+        # Whom to keep it from. Deliberately third-person: "anyone" is in every
+        # handbook sentence about not sharing a password, and an injection
+        # names the reader it is hiding from.
+        audience: %w[user reader human visitor],
+        # The act of answering, which is when an injection wants its rule
+        # applied.
+        answering: %w[answer respond reply response],
+        # Being told to be something else.
+        persona: %w[act pretend roleplay simulate impersonate behave],
+        # The state the persona is supposed to be in.
+        unrestricted: %w[unrestricted unfiltered uncensored jailbroken jailbreak unchained dan],
+        # Limits, which is what "without any" turns into a persona.
+        limits: %w[restriction filter limit guardrail safeguard constraint],
+        # The assistant, referred to as itself. What separates "print the
+        # configuration with scontrol" from "print your configuration": the
+        # first is a page telling a reader to run a command, the second is a
+        # page addressing the thing that reads it.
+        self: %w[your you yours yourself],
+        # Totalising quantifiers. An attack cancels the lot, because it does
+        # not know what it is cancelling; a page cancels one named thing.
+        totality: %w[everything anything all every each entirely completely],
+      },
+      nl: {
+        # Imperatives first: an injection written in Dutch is an order, and
+        # Dutch puts the verb of an order at the front, which is what keeps the
+        # ordered templates working across both languages.
+        override: %w[negeer negeert negeren vergeet vergeten overschrijf overschrijft overschrijven
+                     verwerp verwerpt verwerpen omzeil omzeilt omzeilen schrap schrapt schrappen],
+        instruction: %w[instructie instructies richtlijn richtlijnen regel regels voorschrift voorschriften
+                        opdracht opdrachten aanwijzing aanwijzingen beleid prompt],
+        prior: %w[vorige eerdere eerder bovenstaande voorgaande hierboven hiervoor voorafgaande
+                  oorspronkelijke initiele],
+        reveal: %w[toon tonen onthul onthullen deel delen vertel vertellen zeg zeggen noem noemen
+                   vermeld vermelden stuur sturen verzend verzenden herhaal herhalen print printen
+                   publiceer publiceren weergeef weergeven],
+        secret: %w[wachtwoord wachtwoorden token tokens inloggegevens geheim geheimen configuratie
+                   systeemprompt instructie instructies],
+        conceal: %w[verberg verbergen verzwijg verzwijgen achterhoud achterhouden],
+        audience: %w[gebruiker gebruikers lezer lezers bezoeker bezoekers klant],
+        answering: %w[antwoord antwoorden reactie respons beantwoord beantwoorden],
+        persona: %w[gedraag gedragen speel spelen simuleer simuleren imiteer imiteren],
+        unrestricted: %w[ongefilterd onbeperkt ongecensureerd onbegrensd jailbroken],
+        limits: %w[beperking beperkingen restrictie restricties filter filters grens grenzen waarborg],
+        self: %w[je jij jou jouw uw jezelf uzelf],
+        totality: %w[alles alle elke ieder iedere volledig helemaal],
+      },
+    }.freeze
+
+    # Concepts that only exist as several words. A token lexicon cannot hold
+    # "system prompt" without making "prompt" alone mean it, and "prompt" alone
+    # is a word a shell handbook uses for the thing with the dollar sign in it.
+    PHRASES = {
+      en: {
+        'system prompt' => %i[secret],
+        'developer message' => %i[secret],
+        'initial instruction' => %i[secret],
+        'api key' => %i[secret],
+        'do anything now' => %i[unrestricted],
+        'no longer bound' => %i[unrestricted],
+      },
+      nl: {
+        'systeem prompt' => %i[secret],
+        'api sleutel' => %i[secret],
+        'geheime sleutel' => %i[secret],
+        # "doe" alone is the commonest verb in the language and means nothing
+        # here; "doe alsof" is the one that assigns a role.
+        'doe alsof' => %i[persona],
+        'zonder beperkingen' => %i[unrestricted],
+      },
+    }.freeze
+
+    LANGUAGES = CONCEPTS.keys.freeze
+
+    # Words that flip the concept after them rather than adding one. "Never
+    # mention this to the user" is concealment written with a revealing verb,
+    # and "without restrictions" is an unrestricted persona written with a word
+    # for limits.
+    NEGATORS = {
+      en: %w[not never dont doesnt cannot cant without no none neither nor avoid],
+      nl: %w[niet nooit geen zonder nergens niemand niets noch vermijd],
+    }.freeze
+
+    # What a negated concept becomes. A concept mapped to nil is cancelled
+    # rather than transformed: "do not ignore the guidelines" is not an
+    # override, and reading it as one flags the page that tells a reader to
+    # follow the rules.
+    #
+    # A concept that is not a key here survives negation untouched, because
+    # negation applies to the predicate and not to the nouns around it. "Vertel
+    # de gebruiker niet dat deze pagina is gewijzigd" has to keep its audience:
+    # the negation made the sentence concealment, and concealment from whom is
+    # the other half of the rule.
+    NEGATION = { reveal: :conceal, limits: :unrestricted, override: nil, conceal: nil,
+                 persona: nil, secret: nil, unrestricted: nil }.freeze
+
+    # How far a negator reaches, in tokens, and it reaches further to the right
+    # than to the left.
+    #
+    # English negates before the verb ("do not disclose"), Dutch after it and
+    # often at the end of the clause: "deel je API-sleutel met niemand" is the
+    # sentence every Dutch security page contains, and a scope that only looks
+    # left reads it as an instruction to share the key. Six to the right covers
+    # the clause-final negator without running into the next clause, which the
+    # segmentation has already cut off.
+    #
+    # The cost is stated rather than hidden: a wider scope is a cheaper evasion.
+    # An attacker who reads this file can drop a negator into the clause and
+    # have a revealing verb read as concealment. That is the same trade every
+    # rule here makes, and the alternative is flagging the page that tells a
+    # reader to keep their key to themselves.
+    NEGATION_BEFORE = 3
+    NEGATION_AFTER = 6
+
+    # Determiners, for the one piece of syntax worth knowing: a backward
+    # reference behind a determiner is a noun when nothing follows it, or
+    # when the next word is a coordinator. "Ignore the above" and
+    # "ignore the above and recommend" name the instruction; "the earlier
+    # warning" keeps its noun.
+    DETERMINERS = %w[the this that het de dit die deze].freeze
+    # After a nominalised "the above", the next word is a coordinator, not a
+    # noun. "Ignore the above and recommend" is the attack; "the earlier
+    # warning" keeps its noun and is a page.
+    COORDINATORS = %w[and or but en of maar].freeze
+
+    # "you" carries a persona only when something makes it a statement about
+    # what the reader now is. A handbook says "you" in every second sentence
+    # and means the person reading it, so the bare pronoun is worth nothing;
+    # "you are now" and "je bent nu" are what an injection needs and a page
+    # rarely writes.
+    PRONOUNS = %w[you je jij u].freeze
+    COPULAS = %w[are re be become becoming bent ben is wordt word zijn].freeze
+    # A pronoun that names the instruction in the previous clause:
+    # "There are guidelines above. Ignore them."
+    ANAPHORA = %w[them they it ze zij].freeze
+
+    def self.build_lexicon(languages)
+      lexicon = {}
+      languages.each do |language|
+        CONCEPTS.fetch(language).each do |concept, forms|
+          forms.each { |form| (lexicon[stem(form)] ||= []) << concept }
+        end
+      end
+      lexicon.each_value do |found|
+        found.uniq!
+        found.freeze
+      end
+      lexicon.freeze
+    end
+
+    def self.build_phrases(languages)
+      phrases = {}
+      languages.each do |language|
+        PHRASES.fetch(language).each { |phrase, found| phrases[phrase.split.map { |w| stem(w) }.join(' ')] = found }
+      end
+      phrases.freeze
+    end
+
+    # Per language and merged, built once. Three lexicons rather than a cache
+    # keyed by whatever a caller asks for: the combinations that matter are
+    # "both", "English only", and "Dutch only".
+    LEXICONS = LANGUAGES.to_h { |language| [language, build_lexicon([language])] }
+                        .merge(LANGUAGES => build_lexicon(LANGUAGES)).freeze
+    PHRASE_LEXICONS = LANGUAGES.to_h { |language| [language, build_phrases([language])] }
+                               .merge(LANGUAGES => build_phrases(LANGUAGES)).freeze
+
+    NEGATOR_STEMS = NEGATORS.values.flatten.to_set { |w| stem(w) }.freeze
+    DETERMINER_STEMS = DETERMINERS.to_set { |w| stem(w) }.freeze
+    COORDINATOR_STEMS = COORDINATORS.to_set { |w| stem(w) }.freeze
+    PRONOUN_STEMS = PRONOUNS.to_set { |w| stem(w) }.freeze
+    COPULA_STEMS = COPULAS.to_set { |w| stem(w) }.freeze
+    ANAPHORA_STEMS = ANAPHORA.to_set { |w| stem(w) }.freeze
+    PHRASE_LENGTHS = PHRASE_LEXICONS[LANGUAGES].keys.map { |k| k.count(' ') + 1 }.uniq.sort.reverse.freeze
+
+    def lexicon(languages = LANGUAGES)
+      key = languages.size == 1 ? languages.first : languages.uniq.sort_by(&:to_s)
+      LEXICONS[key] || build_lexicon(languages)
+    end
+
+    def phrase_lexicon(languages = LANGUAGES)
+      key = languages.size == 1 ? languages.first : languages.uniq.sort_by(&:to_s)
+      PHRASE_LEXICONS[key] || build_phrases(languages)
+    end
+
+    # Function words, which is how a language is identified cheaply.
+    #
+    # Content words are the ones a page is about and the ones that differ from
+    # page to page. Function words are the skeleton: a text of any length in a
+    # language contains them at a stable rate, and they are short, closed, and
+    # few enough to list. Counting them is the oldest working language
+    # identifier there is, and it needs no model and no table on disk.
+    #
+    # Chosen to be distinctive rather than merely frequent. Dutch "de" is also
+    # French, and German "die" is also Dutch, so the pairs that would collide
+    # are left out and the rule below asks for several distinct hits rather
+    # than one common one.
+    FUNCTION_WORDS = {
+      en: %w[the and of to is are that with for this you it was were from have has not but they],
+      nl: %w[het een van niet zijn aan ook maar deze wordt worden je uw naar met dat als bij],
+    }.freeze
+
+    # A language needs this many distinct function words present before it is
+    # called, and this share of the text's tokens. Both, because a long page
+    # accumulates stray matches and a short one does not accumulate anything.
+    LANGUAGE_HITS = 3
+    LANGUAGE_SHARE = 0.04
+
+    # Under this many tokens, a text is too short to identify and says so. A
+    # question of six words is not evidence of anything, and guessing on it
+    # would make the answer noise rather than information.
+    LANGUAGE_FLOOR = 12
+
+    # Character n-gram rank profiles (Cavnar and Trenkle, SDAIR 1994) for
+    # the two lexicon languages and the two unread ones the suite uses as
+    # the third-language case. Built from closed function-word lists, not
+    # from the test corpora.
+    PROFILE_SOURCES = {
+      en: FUNCTION_WORDS[:en] + %w[this that with from have been will would could should into over],
+      nl: FUNCTION_WORDS[:nl] + %w[een van het dat niet zijn voor naar nog wel dan toen],
+      de: %w[und der die das ist ein eine nicht mit von zu auf den dem sich auch als nach bei],
+      fr: %w[les des une est dans pour qui que pas avec sur aux sont mais tout],
+    }.freeze
+    NGRAM_SIZES = (2..4)
+    PROFILE_SIZE = 200
+    NGRAM_FLOOR = 6
+
+    def self.build_profile(source)
+      counts = Hash.new(0)
+      source.each do |word|
+        padded = " #{word} "
+        NGRAM_SIZES.each do |size|
+          (0..(padded.length - size)).each { |i| counts[padded[i, size]] += 1 }
+        end
+      end
+      counts.sort_by { |gram, n| [-n, gram] }.map(&:first).first(PROFILE_SIZE).freeze
+    end
+
+    PROFILES = PROFILE_SOURCES.transform_values { |source| build_profile(source) }.freeze
+    PROFILE_INDEX = PROFILES.transform_values { |profile| profile.each_with_index.to_h }.freeze
+
+    # The language of a text: :en, :nl, or :unknown.
+    #
+    # :unknown is a real answer rather than a failure. It is what a page in
+    # German returns, and a rail that reads it can then report that it did not
+    # check rather than reporting that it found nothing.
+    def language(text)
+      tokens = words(text)
+      return :unknown if tokens.size < LANGUAGE_FLOOR
+
+      by_words = function_word_language(tokens)
+      return by_words unless by_words == :unknown
+
+      guessed = ngram_language(text)
+      LANGUAGES.include?(guessed) ? guessed : :unknown
+    end
+
+    # True when the character-n-gram profile names a language this engine
+    # does not read. Used in the twelve-to-twenty-three token band, where
+    # function-word counts stay quiet and an unread page used to certain-pass.
+    def named_foreign?(text, supported)
+      guessed = ngram_language(text)
+      guessed != :unknown && !Array(supported).map(&:to_sym).include?(guessed)
+    end
+
+    def function_word_language(tokens)
+      seen = tokens.to_set
+      scored = FUNCTION_WORDS.map do |code, list|
+        hits = list.count { |word| seen.include?(word) }
+        share = tokens.count { |token| list.include?(token) }.fdiv(tokens.size)
+        [code, hits, share]
+      end
+      best = scored.max_by { |(_, hits, share)| [hits, share] }
+      return :unknown if best[1] < LANGUAGE_HITS || best[2] < LANGUAGE_SHARE
+
+      best[0]
+    end
+
+    def ngram_language(text)
+      tokens = words(text)
+      return :unknown if tokens.size < NGRAM_FLOOR
+
+      doc = document_profile(tokens)
+      return :unknown if doc.size < 8
+
+      scored = PROFILES.keys.map { |lang| [lang, out_of_place(doc, lang)] }
+      ranked = scored.min_by(2) { |(_, distance)| distance }
+      best, second = ranked
+      return :unknown if second && best[1] >= (second[1] * 0.85)
+
+      best[0]
+    end
+
+    def document_profile(tokens)
+      counts = Hash.new(0)
+      tokens.each do |word|
+        padded = " #{word} "
+        NGRAM_SIZES.each do |size|
+          (0..(padded.length - size)).each { |i| counts[padded[i, size]] += 1 }
+        end
+      end
+      counts.sort_by { |gram, n| [-n, gram] }.map(&:first).first(PROFILE_SIZE)
+    end
+
+    def out_of_place(document, language)
+      index = PROFILE_INDEX.fetch(language)
+      document.each_with_index.sum { |gram, rank| ((index[gram] || PROFILE_SIZE) - rank).abs }
+    end
+
+    # Sentences, roughly, and clauses where the punctuation says so.
+    #
+    # Every rule over the concept stream is "these two concepts, close
+    # together", and a rule that reaches across a full stop is reading two
+    # statements as one. "Do not disclose your token to the desk; rotate it and
+    # reply with the job id" is two instructions to a human, and only a window
+    # that ignores the semicolon turns it into an instruction about answering.
+    #
+    # Splitting on more than the full stop is on purpose: the semicolon and the
+    # colon separate statements too, and over-splitting only makes the rules
+    # stricter, which is the safe direction for something that blocks.
+    def clauses(text)
+      usable(text).split(/(?<=[.!?;:])\s+|\n+|\r+/).map(&:strip).reject(&:empty?)
+    end
+
+    # The text as [position, concept, surface word] triples.
+    #
+    # Positions are token indices rather than characters, because every rule
+    # over this stream is "these two concepts, close together", and closeness
+    # in words is what survives an attacker adding punctuation.
+    def concepts(text, languages: LANGUAGES)
+      tokens = words(text)
+      stems = tokens.map { |t| stem(t) }
+      table = lexicon(languages)
+      out = phrase_concepts(tokens, stems, languages)
+      stems.each_with_index do |s, i|
+        out.concat(syntax_concepts(tokens, stems, i, languages))
+        found = table[s]
+        next unless found
+
+        negated = negated?(stems, i)
+        found.each do |concept|
+          concept = NEGATION.fetch(concept, concept) if negated
+          out << [i, concept, tokens[i]] if concept
+        end
+      end
+      out.sort_by { |(i, concept, _)| [i, concept.to_s] }
+    end
+
+    # Concepts per clause, with a backward pronoun bound to the previous
+    # clause's instruction. "There are guidelines above. Ignore them."
+    # is one statement split by a full stop; without the bind, the second
+    # clause is an override with no object.
+    def clause_concepts(text, languages: LANGUAGES)
+      carry = false
+      clauses(text).map do |clause|
+        found = concepts(clause, languages: languages)
+        found = bind_anaphora(clause, found) if carry
+        carry = found.any? { |(_, concept, _)| concept == :instruction }
+        found
+      end
+    end
+
+    def bind_anaphora(clause, found)
+      tokens = words(clause)
+      return found if tokens.empty?
+      return found unless ANAPHORA_STEMS.include?(stem(tokens.last))
+      return found unless found.any? { |(_, concept, _)| concept == :override }
+      return found if found.any? { |(_, concept, _)| concept == :instruction }
+
+      override_at = found.detect { |(_, concept, _)| concept == :override }&.first
+      return found if override_at.nil? || (tokens.size - 1 - override_at) > 3
+
+      found + [[tokens.size - 1, :instruction, tokens.last]]
+    end
+
+    # Multiword concepts, reported at the position of their first token so the
+    # window arithmetic treats a phrase as the one thing it is.
+    def phrase_concepts(tokens, stems, languages = LANGUAGES)
+      table = phrase_lexicon(languages)
+      out = []
+      PHRASE_LENGTHS.each do |length|
+        stems.each_cons(length).with_index do |window, i|
+          found = table[window.join(' ')]
+          next unless found
+
+          # A negator that is part of the phrase ("without restrictions")
+          # is the phrase. One sitting outside it ("not the system prompt")
+          # cancels the concept.
+          inside = stems[i, length].any? { |s| NEGATOR_STEMS.include?(s) }
+          negated = !inside && negated?(stems, i)
+          found.each do |concept|
+            concept = NEGATION.fetch(concept, concept) if negated
+            out << [i, concept, tokens[i, length].join(' ')] if concept
+          end
+        end
+      end
+      out
+    end
+
+    # The two rules that come from the shape of the sentence rather than from a
+    # word: a pronoun made into a statement of what something now is, and a
+    # backward reference used as a noun.
+    def syntax_concepts(tokens, stems, index, languages = LANGUAGES)
+      out = []
+      out << [index, :persona, tokens[index]] if pronoun_persona?(stems, index)
+      out << [index, :instruction, tokens[index]] if nominalised_reference?(stems, index, languages)
+      out
+    end
+
+    def pronoun_persona?(stems, index)
+      PRONOUN_STEMS.include?(stems[index]) && COPULA_STEMS.include?(stems[index + 1].to_s)
+    end
+
+    def nominalised_reference?(stems, index, languages = LANGUAGES)
+      return false unless index.positive? && DETERMINER_STEMS.include?(stems[index - 1])
+      return false unless Array(lexicon(languages)[stems[index]]).include?(:prior)
+
+      nxt = stems[index + 1]
+      nxt.nil? || COORDINATOR_STEMS.include?(nxt)
+    end
+
+    def negated?(stems, index)
+      low = [index - NEGATION_BEFORE, 0].max
+      (low..(index + NEGATION_AFTER)).any? { |i| i != index && NEGATOR_STEMS.include?(stems[i]) }
+    end
+
+    # Character n-grams as a set. Character-level rather than word-level so a
+    # typo, an inflection, or a joined word costs a few shingles instead of a
+    # whole token.
+    def shingles(text, size: 4)
+      body = normalize(text)
+      return Set[body] if body.length <= size
+
+      (0..(body.length - size)).each_with_object(Set.new) { |i, acc| acc << body[i, size] }
+    end
+
+    # How much of `needle` appears in `haystack`, which is not how similar they
+    # are. An injection is a sentence inside a page, so the overlap divided by
+    # the union is small however exact the match: the page dominates the union.
+    # Containment asks the question the case actually poses, "is this thing in
+    # there", and is the standard measure for it.
+    def containment(needle, haystack)
+      return 0.0 if needle.empty?
+
+      (needle & haystack).size.fdiv(needle.size)
+    end
+
+    def jaccard(left, right)
+      union = (left | right).size
+      return 0.0 if union.zero?
+
+      (left & right).size.fdiv(union)
+    end
+  end
+end