@suffolklitlab/alkiln 4.10.0 → 4.10.1-pre-cli

package/package.json

@@ -1,15 +1,13 @@
 {
   "name": "@suffolklitlab/alkiln",
-  "version": "4.10.0",
+  "version": "4.10.1-pre-cli",
   "description": "Integrated automated end-to-end testing with docassemble, puppeteer, and cucumber.",
   "main": "lib/index.js",
   "scripts": {
     "test": "npm run unit && npm run langs_setup && npm run cucumber -- $@",
-    "cucumber_base": "run_cucumber(){ ./node_modules/.bin/cucumber-js \"$@\"; status=$?; ./lib/utils/clean_reports.js; return $status; }; run_cucumber",
-    "cucumber": "run_base(){ npm run cucumber_base -- \"$@\" tests/features/*.feature --retry 1; }; run_base",
-    "cucumber_external": "run_cucumber(){ tmp_path=`pwd`; ( new_path=\"$1\"; shift; cd \"$new_path\"; npm run --prefix \"${tmp_path}\" cucumber_base -- \"$@\" \"$new_path\"/docassemble/*/data/sources/*.feature;) }; run_cucumber",
-    "langs": "npm run langs_setup && ./node_modules/.bin/cucumber-js --require ./lib/index.js tests/features/*_lang*.feature",
-    "langs_setup": "./lib/utils/langs.js 'tests/features/*.feature'",
+    "cucumber": "./lib/run_cucumber.js",
+    "langs": "npm run langs_setup && ./node_modules/.bin/cucumber-js --require ./lib/index.js docassemble/*/data/sources/*_lang*.feature",
+    "langs_setup": "./lib/utils/langs.js 'docassemble/*/data/sources/*.feature'",
     "langs_local": "npm run langs_setup && npm run local",
     "local": "npm run setup && npm run cucumber; npm run takedown",
     "setup": "./lib/docassemble/setup.js",
@@ -36,13 +34,13 @@
   "homepage": "https://suffolklitlab.org/docassemble-AssemblyLine-documentation/docs/automated_integrated_testing/",
   "dependencies": {
     "@axe-core/puppeteer": "4.4.2",
-    "@cucumber/cucumber": "7.3.2",
+    "@cucumber/cucumber": "8.6.0",
     "@faker-js/faker": "^7.3.0",
     "axios": "0.24.0",
     "chai": "4.2.0",
     "cheerio": "1.0.0-rc.10",
-    "dotenv": "8.2.0",
     "deep-equal-in-any-order": "^2.0.0",
+    "dotenv": "8.2.0",
     "mocha": "9.2.0",
     "puppeteer": "13.1.3",
     "qs": "6.10.2",
@@ -52,6 +50,7 @@
   "bin": {
     "alkiln-setup": "./lib/docassemble/setup.js",
     "alkiln-takedown": "./lib/docassemble/takedown.js",
-    "alkiln-langs": "./lib/utils/langs.js"
+    "alkiln-langs": "./lib/utils/langs.js",
+    "alkiln-run": "./lib/run_cucumber.js"
   }
 }

package/setup.py

@@ -0,0 +1,60 @@
+import os
+import sys
+from setuptools import setup, find_packages
+from fnmatch import fnmatchcase
+from distutils.util import convert_path
+
+standard_exclude = ('*.pyc', '*~', '.*', '*.bak', '*.swp*')
+standard_exclude_directories = ('.*', 'CVS', '_darcs', './build', './dist', 'EGG-INFO', '*.egg-info')
+def find_package_data(where='.', package='', exclude=standard_exclude, exclude_directories=standard_exclude_directories):
+    out = {}
+    stack = [(convert_path(where), '', package)]
+    while stack:
+        where, prefix, package = stack.pop(0)
+        for name in os.listdir(where):
+            fn = os.path.join(where, name)
+            if os.path.isdir(fn):
+                bad_name = False
+                for pattern in exclude_directories:
+                    if (fnmatchcase(name, pattern)
+                        or fn.lower() == pattern.lower()):
+                        bad_name = True
+                        break
+                if bad_name:
+                    continue
+                if os.path.isfile(os.path.join(fn, '__init__.py')):
+                    if not package:
+                        new_package = name
+                    else:
+                        new_package = package + '.' + name
+                        stack.append((fn, '', new_package))
+                else:
+                    stack.append((fn, prefix + name + '/', package))
+            else:
+                bad_name = False
+                for pattern in exclude:
+                    if (fnmatchcase(name, pattern)
+                        or fn.lower() == pattern.lower()):
+                        bad_name = True
+                        break
+                if bad_name:
+                    continue
+                out.setdefault(package, []).append(prefix+name)
+    return out
+
+setup(name='docassemble.ALKilnTests',
+      version='1.0.0',
+      description=('A docassemble extension for testing the ALKiln integrated testing framework.'),
+      long_description='# ALKiln support package\r\n\r\nAssembly Line Kiln (ALKiln) is a framework for automatically testing **any** [docassemble](https://docassemble.org/) package using GitHub. ALKiln is being developed as part of the SuffolkLITLab Document Assembly Line project. See [documentation for kiln](https://suffolklitlab.github.io/docassemble-AssemblyLine-documentation/docs/automated_integrated_testing).\r\n\r\nThis package supports ALKiln in different ways.\r\n\r\n## Tool to set up testing\r\n\r\n[Tap here to set up automated integrated testing for your docassemble interview](https://apps-dev.suffolklitlab.org/start/test-setup/).\r\n\r\nThis repo contains a step-by-step form that a developer can use to set up automated integrated testing for **any** docassemble package. It requires, among other things, a docassemble account and a temporary GitHub personal access token with correct permissions. It will add necessary GitHub secrets, make a new branch, push necessary files to that branch, and make a PR with that branch.\r\n\r\n',
+      long_description_content_type='text/markdown',
+      author='plocket',
+      author_email='example@example.com',
+      license='The MIT License (MIT)',
+      url='https://suffolklitlab.org/docassemble-AssemblyLine-documentation/docs/automated_integrated_testing',
+      packages=find_packages(),
+      namespace_packages=['docassemble'],
+      install_requires=['docassemble.ALToolbox>=0.5.1'],
+      zip_safe=False,
+      package_data=find_package_data(where='docassemble/ALKilnTests/', package='docassemble.ALKilnTests'),
+     )
+

package/tests/features/assembly_line.feature

@@ -1,43 +0,0 @@
-@assembly_line
-Feature: Assembly Line package-specific Steps
-
-# In tag names, 'al' is for 'assembly line'
-
-# Note: the first test for this suite is the first test in this file and we're having timeout issues, so we're trying to give the server longer to load.
-
-@fast @al1
-Scenario: I have two name parts
-  Given the max secs for each step is 120
-  Given I start the interview at "AL_tests"
-  And I set the name of "users[0]" to "Uli User"
-  And I tap to continue
-  And I set the address of "users[0]" to "120 Tremont Street, Unit 2, Boston, MA 02108"
-  And I tap to continue
-  And I set the name of "users[1]" to "Uli2 User2"
-  And I set the var "users[i].proxy_var" to "Mixed proxy var step test"
-  And I tap to continue
-  Then the question id should be "end"
-
-@fast @al2
-Scenario: I have three name parts
-  Given I start the interview at "AL_tests"
-  And I set the name of "users[0]" to "Uli Udo User"
-  And I tap to continue
-  And I set the address of "users[0]" to "120 Tremont Street, Unit 2, Boston, MA 02108"
-  And I tap to continue
-  And I set the name of "users[1]" to "Uli2 User2"
-  And I set the var "users[i].proxy_var" to "Mixed proxy var step test"
-  And I tap to continue
-  Then the question id should be "end"
-
-@fast @al3
-Scenario: I have four name parts
-  Given I start the interview at "AL_tests"
-  And I set the name of "users[0]" to "Uli Udo User Sr"
-  And I tap to continue
-  And I set the address of "users[0]" to "120 Tremont Street, Unit 2, Boston, MA 02108"
-  And I tap to continue
-  And I set the name of "users[1]" to "Uli2 User2"
-  And I set the var "users[i].proxy_var" to "Mixed proxy var step test"
-  And I tap to continue
-  Then the question id should be "end"

package/tests/features/establishing_steps.feature

@@ -1,62 +0,0 @@
-@establishing
-Feature: Establishing steps
-
-Steps that establish things about the state of the test
-# In tag names, 'e' is for 'establishing'
-
-@slow @e1 @wait
-Scenario: I am able to set a custom wait time that allows steps to run longer than the half minute default
-  Given I start the interview at "all_tests"
-  And the max seconds for each step in this scenario is 40
-  And I wait 35 seconds
-
-@slow @e2 @wait
-Scenario: I am able to set a custom wait time before an interview has been loaded
-  Given the max seconds for each step in this scenario is 40
-  And I start the interview at "all_tests"
-  And I wait 35 seconds
-
-@fast @e3 @urlargs
-Scenario: Interview name includes url args
-  Given I start the interview at "url_args.yml&from=theinternets&random=zoo"
-  Then I should see the phrase "zoo"
-
-@slow @e4 @wait_first
-Scenario: I can wait as a first step in a test
-  Given I wait 1 second
-  
-@slow @e5 @wait_first
-Scenario: Interview name includes url args with a wait
-  Given I wait 1 second
-  Then I start the interview at "url_args.yml&from=theinternets&random=zoo"
-  Then I should see the phrase "zoo"
-
-@fast @e6 @signin
-Scenario: I sign in
-  Given I sign in with the email "USER1_EMAIL" and the password "USER1_PASSWORD"
-  And I start the interview at "all_tests"
-
-@fast @e7 @arbitraryurl
-Scenario: I want to go to a full arbitrary url
-  Given the max seconds for each step in this scenario is 5
-  And the Scenario report should include:
-  """
-  Trying to load the interview
-  """
-  Given I start the interview at "https://apps-test.suffolklitlab.org/start/demo/questions"
-  Then I should see the phrase "What language do you speak?"
-
-# WARNING: This Scenario may fail if the file moves or changes too much
-@fast @failing @e8 @rfe8 @arbitraryurl
-Scenario: Fail with no interview at arbitrary url
-  Given the final Scenario status should be "failed"
-  And the Scenario report should include:
-  """
-  Trying to load the interview
-  """
-  And the Scenario report should include:
-  """
-  ALKiln could not find any interview question page
-  """
-  And the max seconds for each step in this scenario is 5
-  Then I start the interview at "https://apps-test.suffolklitlab.org/list"

package/tests/features/example_test.feature

@@ -1,11 +0,0 @@
-@example_feature_tag
-Feature: An example test
-
-The tests in this file should cover
-- [x] Provide a template people can use to make their own tests.
-- [ ] Direct people to more documentation.
-- [ ] Find the ultimate question to life, the universe, and everything.
-
-@example_scenario_tag
-Scenario: Server finishes reloading
-  Given I start the interview at "test"

package/tests/features/interactive_steps.feature

@@ -1,166 +0,0 @@
-@interactive
-Feature: Interactive steps
-
-# In tag names, 'i' is for 'interactive'
-
-@slow @i1
-Scenario: I set various values
-  Given I start the interview at "all_tests"
-  Then the question id should be "upload files"
-  When I set the var "upload_files_visible" to "some_png_1.png, some_png_2.png"
-  And I set the var "show_upload" to "True"
-  And I upload "some_png_2.png,some_png_1.png" to "upload_files_hidden"
-  And I tap to continue
-  # Next page
-  Then the question id should be "group of complex fields"
-  And I set the var "double_quote_dict['double_quote_key']['dq_two']" to "true"
-  And I set the var "single_quote_dict['single_quote_key']['sq_two']" to "true"
-  And I tap to continue
-  Then the question id should be "direct standard fields"
-  # Next page
-  When I set the var "checkboxes_yesno" to "True"
-  And I set the var "checkboxes_other['checkbox_other_opt_1']" to "true"
-  And I set the var "dropdown_test" to "dropdown_opt_2"
-  And I set the var "radio_yesno" to "False"
-  And I set the var "radio_other" to "radio_other_opt_3"
-  And I set the var "text_input" to "Regular text input field value"
-  And I set the var "textarea" to "Multiline text\narea value"
-  And I set the var "date_input" to "today - 1"
-  When I set the var "direct_standard_fields" to "True"
-  # Next page
-  Then the question id should be "showifs"
-  When I set the var "show_2" to "True"
-  And I set the var "show_3" to "True"
-  And I set the var "showif_checkbox_yesno" to "true"
-  And I set the var "showif_checkboxes_other['showif_checkboxes_nota_2']" to "true"
-  And I set the var "showif_checkboxes_other['showif_checkboxes_nota_3']" to "true"
-  And I set the var "showif_yesnoradio" to "True"
-  And I set the var "showif_radio_other" to "showif_radio_multi_2"
-  And I set the var "showif_text_input" to "Show if text input value"
-  And I set the var "showif_textarea" to "Show if\nmultiline text\narea value"
-  And I set the var "showif_dropdown" to "showif_dropdown_1"
-  When I tap to continue
-  # Next page
-  Then the question id should be "object checkboxes"
-  And I set the var "object_checkboxes_test['obj_chkbx_opt_1']" to "true" 
-  And I set the var "object_dropdown" to "obj_opt_2"
-  When I tap to continue
-  # Next page
-  Then the question id should be "buttons yesnomaybe"
-  When I set the var "buttons_yesnomaybe" to "None"
-  # Next page
-  Then the question id should be "buttons other"
-  When I set the var "buttons_other" to "button_3"
-  # Next page
-  Then the question id should be "button continue"
-  When I set the var "button_continue" to "True"
-  # Next page
-  Then the question id should be "screen features"
-  When I tap to continue
-  # Next page
-  Then the question id should be "proxy vars"
-  When I set the var "x[i].name.first" to "Proxyname1"
-  And I tap to continue
-  # Next page
-  Then the question id should be "is there another generic"
-  When I set the var "x.there_is_another" to "True"
-  # Next page
-  Then the question id should be "proxy vars"
-  When I set the var "x[i].name.first" to "Proxyname2"
-  And I tap to continue
-  # Next page
-  Then the question id should be "is there another generic"
-  When I set the var "x.there_is_another" to "False"
-  # Next page
-  Then the question id should be "first signature"
-  When I sign
-  And I tap to continue
-  # Next page
-  Then the question id should be "second signature"
-  When I sign
-  And I tap to continue
-  # Next page
-  Then the question id should be "simple doc"
-  When I download "simple-doc.pdf"
-  And I tap to continue
-  # Next page
-  Then the question id should be "button event action"
-  And I tap to continue
-  # Next page
-  Then the question id should be "single-dropdown-field"
-  And I set the var "single_dropdown_field" to "email"
-  And I tap to continue
-  # Next page
-  Then the question id should be "the end"
-
-@fast @i2 @secret
-Scenario: handles settings from Github secrets
-  Given the final Scenario status should be "failed"
-  And the Scenario report should include:
-  """
-  The GitHub SECRET "SECRET_NOT_THERE" doesn't exist
-  """
-  Given I start the interview at "test_secret_vars"
-  When I set the var "first_text_entry" to the secret "SECRET_VAR1"
-  And I tap to continue
-  Then I see the phrase "secret-var1-value"
-  When I tap to continue
-  And I set the var "second_text_entry" to the github secret "SECRET_VAR2"
-  And I tap to continue
-  Then I see the phrase "secret-var2-value"
-  When I tap to continue
-  And I set the variable "third_text_entry" to secret "SECRET_NOT_THERE"
-
-@i3 @tap-elements @tabs
-Scenario: tap elements
-  Given I start the interview at "test_taps"
-  And I tap the "Tests-first_template-tab" tab
-  Then I see the phrase "Mechanics"
-  And I tap the "Tests-second_template-tab" tab
-  Then I see the phrase "villify"
-  And I tap the "Tests-third_template-tab" tab
-  Then I see the phrase "museum"
-  And I tap the "#special_event" element and wait 1 second
-  Then I see the phrase "Portishead"
-
-@i4 @tap-elements @tabs @error
-Scenario: tap element with an error
-  Given the final Scenario status should be "failed"
-  And the Scenario report should include:
-  """
-  is there a typo?
-  """
-  Given I start the interview at "test_taps"
-  And I tap the "Tests-not_there-tab" tab
-
-@i5 @decoding-test @object-multiselect
-Scenario: Base64 encoded corner cases are decoded correctly
-  Given I start the interview at "test_decoding"
-  And I get to "end screen" with this data:
-    | var | value | trigger |
-    | object_multiselect_test['obj_opt_1'] | True | |
-    | object_multiselect_test['obj_opt_3'] | True | |
-    | multiselect_test['hi'] | True | |
-    | multiselect_test['dear'] | True | |
-    | object_radio_field | object_radio_choice_one | |
-    | object_checkboxes_test['obj_chkbx_opt_1'] | True | |
-    | object_dropdown | obj_opt_2 | |
-    | checkbox_dict['single_quote_key']['🁳🧴🯍🥫🅍🗎🹇🗷🌲🯵'] | True | |
-
-# This test should fail because it's target id is never found, but that also means
-# it has succeeded because it didn't try to keep looking for the id by pressing 'restart'
-# to continue.
-@fast @i6
-Scenario: Fails as it doesn't try to 'continue' with a restart button
-  Given the final Scenario status should be "failed"
-  And the Scenario report should include:
-  """
-  The test got stuck on "kickout-screen"
-  """
-  Given the max seconds for each step in this scenario is 10
-  Given I start the interview at "test_kickout"
-  And I get to "doesnt exist" with this data:
-    | var | value | trigger |
-    | user_choice | wrong | |
-  
-

package/tests/features/observation_steps.feature

@@ -1,153 +0,0 @@
-@observation
-Feature: Observational steps
-
-# In tag names, 'o' is for 'observational'
-
-@fast @o1
-Scenario: I observe things on a single page when I arrive
-  Given I start the interview at "all_tests"
-  And I tap to continue
-  Then the question id should be "group of complex fields"
-  And I should see the phrase "Some complex fields"
-  And I SHOULD see the phrase "Some complex fields"
-  And I should not see the phrase "zzzzzzz"
-  And I should NOT see the phrase "zzzzzzz"
-  And an element should have the id "daform"
-
-@fast @o2
-Scenario: I see user errors.
-#  They're everywhere.  Some don't even know they are errors.
-  Given I start the interview at "all_tests"
-  And I tap to continue
-  And I tap to continue
-  And I tap to continue
-  Then I can't continue
-  And I will be told an answer is invalid
-
-@fast @o3
-Scenario: I can include .yml in the filename
-  Given I start the interview at "all_tests.yml"
-
-@slow @o4
-Scenario: I check navigation
-  Given I start the interview at "all_tests"
-  And I tap to continue
-  And I get to "showifs" with this data:
-    | var | value | trigger |
-    | double_quote_dict["double_quote_key"]['dq_two'] | true |  |
-    | checkboxes_other['checkbox_other_opt_1'] | true |  |
-    | dropdown_test | dropdown_opt_2 | |
-    | radio_yesno | False | false |
-    | radio_other | radio_other_opt_3 | |
-    | single_quote_dict['single_quote_key']['sq_two'] | true |  |
-    | text_input | Regular text input field value | |
-    | textarea | Multiline text\narea value | |
-    | date_input | today | |
-  Then I arrive at the next page
-  Then I get to "screen features" with this data:
-    | var | value | trigger |
-    | object_checkboxes_test["obj_chkbx_opt_1"] | True | |
-    | object_dropdown | obj_opt_2 | |
-    | button_continue | True |  |
-    | buttons_other | button_2 |  |
-    | buttons_yesnomaybe | True |  |
-  And I should see the link to "http://ecosia.org/"
-
-@fast @o4
-Scenario: Test "Then I don't continue" with a single quote
-  Given I start the interview at "all_tests"
-  And I tap to continue
-  And I tap to continue
-  And I tap to continue
-  Then I don't continue
-  And I will be told an answer is invalid
-
-@fast @o5
-Scenario: Test "Then I cannot continue"
-  Given I start the interview at "all_tests"
-  And I tap to continue
-  And I tap to continue
-  And I tap to continue
-  Then I cannot continue
-  And I will be told an answer is invalid
-
-@fast @o6
-Scenario: Test "Then I do not continue"
-  Given I start the interview at "all_tests"
-  And I tap to continue
-  And I tap to continue
-  And I tap to continue
-  Then I do not continue
-  And I will be told an answer is invalid
-
-@fast @o7
-Scenario: Test "Then I can’t continue" with an apostrophe
-  Given I start the interview at "all_tests"
-  And I tap to continue
-  And I tap to continue
-  And I tap to continue
-  Then I can’t continue
-  And I will be told an answer is invalid
-
-@fast @o8
-Scenario: Test "Then I don’t continue" with an apostrophe
-  Given I start the interview at "all_tests"
-  And I tap to continue
-  And I tap to continue
-  And I tap to continue
-  Then I don’t continue
-  And I will be told an answer is invalid
-
-@fast @o9 @table @json
-Scenario: I can match JSON page var to str
-  Given I start the interview at "all_tests.yml"
-  And I get to "showifs" with this data:
-    | var | value | trigger |
-    | double_quote_dict["double_quote_key"]['dq_two'] | true |  |
-    | checkboxes_other['checkbox_other_opt_1'] | true |  |
-    | dropdown_test | dropdown_opt_2 | |
-    | radio_yesno | False | false |
-    | radio_other | radio_other_opt_3 | |
-    | single_quote_dict['single_quote_key']['sq_two'] | true |  |
-    | text_input | Regular text input field value | |
-    | textarea | Multiline text\narea value | |
-    | date_input | today | |
-  Then the text in the JSON variable "dropdown_test" should be
-  """
-  dropdown_opt_2
-  """
-
-# Maybe this one should be in the report tests (as well?)
-@fast @o10 @json
-Scenario: I get the page's JSON
-  Given I start the interview at "all_tests"
-  Then I get the page's JSON variables and values
-
-@fast @o11 @screenshot
-Scenario: I take a screenshot
-  Given I start the interview at "all_tests"
-  Then I take a screenshot
-  Then I take a screenshot named "some-screenshot"
-
-@slow @o12 @accessibility @a11y
-Scenario: I check the pages for accessibility
-  Given I start the interview at "all_tests"
-  And I check all pages for accessibility issues
-  And I tap to continue
-  Then I get to "screen features" with this data:
-    | var | value | trigger |
-    | double_quote_dict["double_quote_key"]['dq_two'] | true |  |
-    | checkboxes_other['checkbox_other_opt_1'] | true |  |
-    | dropdown_test | dropdown_opt_2 | |
-    | radio_yesno | False | false |
-    | radio_other | radio_other_opt_3 | |
-    | object_checkboxes_test["obj_chkbx_opt_1"] | True | |
-    | object_dropdown | obj_opt_2 | |
-    | object_select_test | obj_chkbx_opt_2 | |
-    | single_quote_dict['single_quote_key']['sq_two'] | true |  |
-    | text_input | Regular text input field value | |
-    | textarea | Multiline text\narea value | |
-    | date_input | today | |
-    | button_continue | True |  |
-    | buttons_other | button_2 |  |
-    | buttons_yesnomaybe | True |  |

package/tests/features/random_input.feature

@@ -1,31 +0,0 @@
-@random_input
-Feature: Assembly Line package-specific Steps
-
-# In tag names, 'ri' is for 'random input'
-
-@fast @ri1 @random
-Scenario: I fill in random input till the end
-  Given I start the interview at "test_random_input"
-  And I answer randomly for at most 20 pages
-  Then the question id should be "end"
-
-@fast @ri2 @random
-Scenario: I fill in random input for 1 page
-  Given I start the interview at "test_random_input"
-  And I answer randomly for at most 1 page
-
-# Should create two unique folders for random step screenshots which
-# must be checked manually
-@fast @ri3 @random
-Scenario: I answer randomly till the end twice
-  Given I start the interview at "test_random_input"
-  And I answer randomly for at most 50 pages
-  Given I start the interview at "test_random_input"
-  And I answer randomly for at most 50 pages
-
-@fast @ri4 @random @error
-Scenario: Fail with error page from random input
-  Given the final Scenario status should be "failed"
-  Given I start the interview at "test_missing_var_error_screen"
-  And the max seconds for each step in this scenario is 10
-  And I answer randomly for at most 3 pages