@concordance-wiki/nlp 0.1.0

package/locales/en/stopwords.txt

@@ -0,0 +1,536 @@
+# Default English stopwords of the `en` language pack: one word per line, lowercase.
+#
+# A stopword is a word that names nothing on its own in any corpus: articles,
+# prepositions, pronouns, conjunctions, the auxiliaries and the light verbs in
+# every form, the adverbs and connectors of prose, quantifiers, numerals, and the
+# few nouns that only count or place ("thing", "way", "time"). A word that could be
+# a business term somewhere ("contract", "rule", "source", "part", "case", "type")
+# or that ends a technical compound ("cold start", "change set", "entry point")
+# never goes here: a project adds it to a file listed under `inference.stopwords`.
+#
+# A title or alias that is a stopword never enters the recognition dictionary, and
+# the keyword discovery drops every n-gram starting or ending with one. Stopwords
+# are compared on the same form as the texts: plurals reduced, so "things" and
+# "thing" are one entry to the engine. One section per class, alphabetical inside a
+# section. The pieces of a contraction ("it's", "don't") are listed as the tokeniser
+# cuts them at the apostrophe: "s", "t", "don".
+# Project files listed under `inference.stopwords` are added on top of this list.
+
+# articles and determiners
+a
+all
+an
+another
+any
+both
+certain
+each
+either
+enough
+entire
+every
+few
+fewer
+half
+least
+less
+many
+more
+most
+much
+neither
+no
+none
+other
+others
+own
+same
+several
+some
+such
+the
+these
+this
+those
+various
+whole
+
+# conjunctions
+although
+and
+as
+because
+but
+else
+for
+if
+nor
+once
+or
+since
+so
+than
+then
+though
+unless
+until
+whatever
+whenever
+whereas
+wherever
+whether
+which
+whichever
+while
+whoever
+yet
+
+# prepositions
+about
+above
+across
+after
+against
+along
+amid
+among
+amongst
+around
+at
+before
+behind
+below
+beneath
+beside
+besides
+between
+beyond
+by
+concerning
+despite
+down
+during
+except
+from
+in
+inside
+into
+near
+of
+off
+on
+onto
+out
+outside
+over
+per
+regarding
+through
+throughout
+till
+to
+toward
+towards
+under
+up
+upon
+via
+with
+within
+without
+
+# pronouns
+anybody
+anyone
+anything
+everybody
+everyone
+everything
+he
+her
+hers
+herself
+him
+himself
+his
+how
+i
+it
+its
+itself
+me
+mine
+my
+myself
+nobody
+nothing
+one
+ones
+oneself
+our
+ours
+ourselves
+she
+somebody
+someone
+something
+that
+their
+theirs
+them
+themselves
+they
+us
+we
+what
+when
+where
+who
+whom
+whose
+why
+you
+your
+yours
+yourself
+yourselves
+
+# contractions
+aren
+cannot
+couldn
+d
+didn
+doesn
+don
+hadn
+hasn
+haven
+isn
+ll
+m
+mustn
+re
+s
+shouldn
+t
+ve
+wasn
+weren
+won
+wouldn
+
+# auxiliaries and modal verbs
+am
+are
+be
+been
+being
+can
+could
+did
+do
+does
+doing
+done
+had
+has
+have
+having
+is
+may
+might
+must
+ought
+shall
+should
+was
+were
+will
+would
+
+# light verbs
+able
+allow
+allowed
+allowing
+allows
+became
+become
+becomes
+becoming
+came
+come
+comes
+coming
+gave
+get
+gets
+getting
+give
+given
+gives
+giving
+go
+goes
+going
+gone
+got
+gotten
+keep
+keeping
+keeps
+kept
+knew
+know
+knowing
+known
+knows
+let
+lets
+letting
+made
+make
+makes
+making
+need
+needed
+needing
+needs
+put
+puts
+putting
+said
+saw
+say
+saying
+says
+see
+seeing
+seem
+seemed
+seeming
+seems
+seen
+sees
+take
+taken
+takes
+taking
+tell
+telling
+tells
+told
+took
+tried
+tries
+try
+trying
+unable
+use
+used
+uses
+using
+want
+wanted
+wanting
+wants
+went
+
+# adverbs and connectors
+actually
+again
+ago
+alike
+almost
+alone
+already
+also
+altogether
+always
+anyway
+anywhere
+apart
+away
+back
+basically
+certainly
+clearly
+currently
+definitely
+directly
+easily
+elsewhere
+entirely
+especially
+essentially
+even
+eventually
+ever
+exactly
+far
+finally
+forth
+frequently
+further
+generally
+hardly
+hence
+here
+however
+immediately
+indeed
+initially
+instead
+just
+largely
+later
+likely
+likewise
+mainly
+maybe
+meanwhile
+merely
+moreover
+mostly
+namely
+naturally
+nearly
+necessarily
+never
+nevertheless
+nonetheless
+normally
+not
+notably
+now
+nowadays
+obviously
+occasionally
+often
+only
+otherwise
+overall
+particularly
+partly
+perhaps
+possibly
+probably
+properly
+quite
+rarely
+rather
+really
+recently
+regardless
+relatively
+roughly
+seldom
+simply
+slightly
+somehow
+sometimes
+somewhat
+somewhere
+soon
+specifically
+still
+sure
+surely
+there
+therefore
+thus
+together
+too
+totally
+truly
+twice
+typically
+ultimately
+usually
+very
+well
+wholly
+widely
+yes
+
+# ordinals and time
+day
+days
+first
+following
+former
+last
+latter
+moment
+month
+months
+next
+previous
+today
+tomorrow
+week
+weeks
+year
+years
+yesterday
+
+# numerals
+eight
+five
+four
+hundred
+nine
+seven
+six
+ten
+thousand
+three
+two
+
+# generic nouns
+example
+examples
+kind
+kinds
+level
+levels
+manner
+matter
+number
+numbers
+rest
+sort
+sorts
+thing
+things
+time
+times
+way
+ways
+
+# generic adjectives
+absent
+actual
+available
+bad
+best
+better
+common
+different
+false
+general
+good
+main
+new
+old
+particular
+possible
+present
+real
+similar
+single
+true
+usual
+worse
+worst

package/locales/en/suffixes.txt

@@ -0,0 +1,7 @@
+# Endings of the inflected forms of English: a candidate expression whose word ends with one
+# is more often an adverb or a verb form of the prose than a term, and its confidence takes
+# a soft penalty, never an elimination ("meeting", "feed" are terms). One suffix per line,
+# the leading hyphen optional; compared on the comparison form of the words.
+-ly
+-ing
+-ed

package/locales/fr/pack.yaml

@@ -0,0 +1,15 @@
+# Language pack: everything the engine needs to know about a language, as data.
+locale: fr
+language: Français
+apostrophes: ["’", "ʼ"]
+collation: { sensitivity: base, numeric: true }
+# The first matching rule wins: -eaux comes before -aux so that "bateaux" gives "bateau".
+# Other words in -x ("prix", "voix", "taux") are invariant and match no rule.
+plural:
+  - { ending: eaux, singular: eau, min_length: 6 }
+  - { ending: aux, singular: al, min_length: 6 }
+  - { ending: eux, singular: eu, min_length: 4 }
+  - { ending: oux, singular: ou, min_length: 5 }
+  # A word in -ss is a singular ("stress"): this rule keeps it from the -s rule below.
+  - { ending: ss, singular: ss }
+  - { ending: s, singular: "", min_length: 4 }