ccs-digitalmarketplace-frameworks 3.3.0 → 3.4.0

package/.github/workflows/test.yml

@@ -40,6 +40,7 @@ jobs:
           - scripts/generate-assessment-schema.py g-cloud-14
           - scripts/generate-search-config.py g-cloud-14 services --output-path=/tmp
           - scripts/generate-validation-schemas.py --output-path=/tmp
+          - scripts/generate-filter-schemas.py --output-path=/tmp
 
     steps:
       - uses: actions/checkout@v4

package/README.md

@@ -186,6 +186,9 @@ Don't forget that you may also need to generate schemas into the API's `json_sch
 `scripts/generate-validation-schemas.py` script in this repo, or similarly update the Search API using the
 `scripts/generate-search-config.py` script.
 
+You may also need to generate filter schemas into the Supplier API's `filters` directory using the
+`scripts/generate-filter-schemas.py` script in this repo.
+
 Running the tests
 -----------------
 

package/frameworks/digital-outcomes-and-specialists-7/manifests/edit_service.yml

@@ -240,3 +240,8 @@
   editable: False
   questions:
   - whatTypesOfUserResearchStudio
+
+- name: User research studios
+  editable: False
+  questions:
+  - multiqTradeOrIndustryBodies

package/frameworks/digital-outcomes-and-specialists-7/manifests/edit_service_as_admin.yml

@@ -268,3 +268,8 @@
   editable: True
   questions:
   - whatTypesOfUserResearchStudio
+
+- name: User research studios
+  editable: True
+  questions:
+  - multiqTradeOrIndustryBodies

package/frameworks/digital-outcomes-and-specialists-7/manifests/edit_submission.yml

@@ -312,3 +312,8 @@
   editable: True
   questions:
   - whatTypesOfUserResearchStudio
+
+- name: User research studios
+  editable: True
+  questions:
+  - multiqTradeOrIndustryBodies

package/frameworks/digital-outcomes-and-specialists-7/questions/services/icoRegistered.yml

@@ -1,3 +1,4 @@
+name: Registered with the Information Commissioner's Office (ICO)
 question: Are you registered with the Information Commissioner's Office (ICO)?
 
 depends:

package/frameworks/digital-outcomes-and-specialists-7/questions/services/icoRegistrationReference.yml

@@ -1,3 +1,4 @@
+name: Registration reference
 question: Enter your registration reference
 
 depends:
@@ -12,5 +13,5 @@ hidden: true
 validations:
   - name: answer_required
     message: Enter your registration reference
-  - name: under_word_limit
+  - name: under_10_words
     message: "Your answer must be no longer than 10 words"

package/frameworks/digital-outcomes-and-specialists-7/questions/services/multiqIcoRegistered.yml

@@ -1,5 +1,5 @@
 name: Registered with the Information Commissioner's Office (ICO)
-question: Registered with the Information Commissioner's Office (ICO)
+question: ""
 
 depends:
   - "on": lot

package/frameworks/digital-outcomes-and-specialists-7/questions/services/multiqOtherTradeOrIndustryBodies.yml

@@ -1,5 +1,5 @@
 name: Member of any other trade or industry bodies
-question: Member of any other trade or industry bodies
+question: ""
 
 depends:
   - "on": lot

package/frameworks/digital-outcomes-and-specialists-7/questions/services/multiqTradeOrIndustryBodies.yml

@@ -0,0 +1,17 @@
+name: Member of any trade or industry bodies
+question: ""
+
+depends:
+  - "on": lot
+    being:
+      - user-research-studios-and-participants
+
+depends_on_follow_up:
+  whatTypesOfUserResearchStudio:
+    - virtual-studio
+
+type: multiquestion
+
+questions:
+  - tradeOrIndustryBodies
+  - tradeOrIndustryBodiesOther

package/frameworks/digital-outcomes-and-specialists-7/questions/services/otherTradeOrIndustryBodies.yml

@@ -1,4 +1,6 @@
+name: Member of any other trade or industry bodies
 question: Are you a member of any other trade or industry bodies?
+hint: Select all that apply
 
 depends:
   - "on": lot

package/frameworks/digital-outcomes-and-specialists-7/questions/services/otherTradeOrIndustryBodiesOther.yml

@@ -1,4 +1,5 @@
-question: Specify any other trade or industry bodies you a member of
+name: Other trade or industry bodies you are a member of
+question: Specify any other trade or industry bodies you are a member of
 
 depends:
   - "on": lot
@@ -11,8 +12,8 @@ hidden: true
 
 validations:
   - name: answer_required
-    message: Enter any other trade or industry bodies you a member of
-  - name: under_word_limit
+    message: Enter any other trade or industry bodies you are a member of
+  - name: under_200_words
     message: "Your answer must be no more than 200 words"
   - name: under_character_limit
     message: "Your answer must be no more than 2000 characters"

package/frameworks/digital-outcomes-and-specialists-7/questions/services/tradeOrIndustryBodies.yml

@@ -0,0 +1,33 @@
+name: Member of any trade or industry bodies
+question: Are you a member of any trade or industry bodies?
+hint: Select all that apply
+
+depends:
+  - "on": lot
+    being:
+      - user-research-studios-and-participants
+
+depends_on_follow_up:
+  whatTypesOfUserResearchStudio:
+    - virtual-studio
+
+type: checkboxes
+
+options:
+  - label: Association of Qualitative Research (AQR)
+    value: aqr
+  - label: Market Research Society (MRS)
+    value: mrs
+  - label: Viewing Facilities Association (VFA)
+    value: vfa
+  - label: Other
+    value: other
+
+followup:
+  tradeOrIndustryBodiesOther:
+    - other
+
+validations:
+  -
+    name: answer_required
+    message: Select if you are a member of any trade or industry bodies

package/frameworks/digital-outcomes-and-specialists-7/questions/services/tradeOrIndustryBodiesOther.yml

@@ -0,0 +1,19 @@
+name: Other trade or industry bodies you are a member of
+question: Specify any other trade or industry bodies you are a member of
+
+depends:
+  - "on": lot
+    being:
+      - user-research-studios-and-participants
+
+type: textbox_large
+max_length_in_words: 200
+hidden: true
+
+validations:
+  - name: answer_required
+    message: Enter any other trade or industry bodies you are a member of
+  - name: under_200_words
+    message: "Your answer must be no more than 200 words"
+  - name: under_character_limit
+    message: "Your answer must be no more than 2000 characters"

package/frameworks/digital-outcomes-and-specialists-7/questions/services/whatTypesOfUserResearchStudio.yml

@@ -18,6 +18,14 @@ depends_on_follow_up:
   whatUserResearchServices:
     - user-research-studios
 
+dependent_follow_up:
+  tradeOrIndustryBodies:
+    - virtual-studio
+
+dependent_follow_up_followup:
+  tradeOrIndustryBodiesOther:
+    - virtual-studio
+
 validations:
   -
     name: answer_required

package/frameworks/digital-outcomes-and-specialists-7/questions/services/whatUserResearchServices.yml

@@ -33,12 +33,16 @@ dependent_follow_up:
     - user-research-participants
   whatTypesOfUserResearchStudio:
     - user-research-studios
+  tradeOrIndustryBodies:
+    - user-research-studios
 
 dependent_follow_up_followup:
   otherTradeOrIndustryBodiesOther:
     - user-research-participants
   icoRegistrationReference:
     - user-research-participants
+  tradeOrIndustryBodiesOther:
+    - user-research-studios
 
 validations:
   -

package/frameworks/g-cloud-14/metadata/following_framework.yml

@@ -1,4 +1,4 @@
 framework:
-  coming: '2024'
+  coming: '2025'
   name: G-Cloud 15
   slug: g-cloud-15

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ccs-digitalmarketplace-frameworks",
-  "version": "3.3.0",
+  "version": "3.4.0",
   "description": "Data files for Digital Marketplace’s procurement frameworks",
   "repository": "git@github.com:Crown-Commercial-Service/ccs-digitalmarketplace-frameworks",
   "author": "enquiries@digitalmarketplace.service.gov.uk"

package/requirements-dev.txt

@@ -115,7 +115,7 @@ govuk-country-register==0.5.0
     # via
     #   -r requirements.txt
     #   ccs-digitalmarketplace-utils
-hypothesis==6.115.0
+hypothesis==6.115.3
     # via -r requirements-dev.in
 idna==3.7
     # via

package/schema_generator/filter.py

@@ -0,0 +1,151 @@
+import os
+import json
+import time
+from dmcontent import ContentLoader
+from collections import OrderedDict
+
+G_CLOUD_LOTS = [
+    ('cloud-hosting', 'Cloud Hosting'),
+    ('cloud-software', 'Cloud Software'),
+    ('cloud-support', 'Cloud Support')
+]
+
+FRAMEWORKS_AND_LOTS = {
+    "g-cloud": [
+        {
+            'framework_slug': 'g-cloud-12',
+            'framework_name': "G-Cloud 12",
+            'lots': G_CLOUD_LOTS
+        },
+        {
+            'framework_slug': 'g-cloud-13',
+            'framework_name': "G-Cloud 13",
+            'lots': G_CLOUD_LOTS,
+        },
+        {
+            'framework_slug': 'g-cloud-14',
+            'framework_name': "G-Cloud 14",
+            'lots': G_CLOUD_LOTS,
+        },
+    ]
+}
+
+
+def _get_schema_title_and_slugs():
+    titles_and_slugs = []
+    for _family, framework_list in FRAMEWORKS_AND_LOTS.items():
+        for framework in framework_list:
+            for lot_slug, _lot_name in framework['lots']:
+                titles_and_slugs.append(
+                    (
+                        framework['framework_slug'],
+                        lot_slug
+                    )
+                )
+    return titles_and_slugs
+
+
+SCHEMAS = _get_schema_title_and_slugs()
+
+
+def _get_filter_value_from_question_option(option):
+    """
+    Processes `option` into a search term value suitable for the Search API.
+    :return: normalized string value
+    """
+    return (option.get('value') or option.get('label', '')).lower().replace(',', '')
+
+
+def _sections_for_lot(lot, builder, all_lots=None):
+    if all_lots is None:
+        all_lots = []
+    if lot is None or lot == 'all':
+        for lot_slug in [x['slug'] for x in all_lots]:
+            builder = builder.filter({'lot': lot_slug})
+    else:
+        builder = builder.filter({'lot': lot})
+
+    return builder.sections
+
+
+def _filters_for_lot(lot, builder, all_lots=None):
+    if all_lots is None:
+        all_lots = []
+    sections = _sections_for_lot(lot, builder, all_lots=all_lots)
+    lot_filters = OrderedDict()
+
+    for section in sections:
+        section_filter = {
+            "label": section["name"],
+            "slug": section["slug"],
+            "filters": [],
+        }
+        for question in section["questions"]:
+            section_filter["filters"].extend(
+                _filters_for_question(question)
+            )
+
+        lot_filters[section.slug] = section_filter
+
+    return lot_filters
+
+
+def _filters_for_question(question):
+    question_filters = []
+    if question['type'] == 'boolean':
+        question_filters.append({
+            'label': question.get('filter_label') or question.get('name') or question['question'],
+            'name': question['id'],
+            'id': question['id'],
+            'value': 'true',
+        })
+
+    elif question['type'] in ['checkboxes', 'radios', 'checkbox_tree']:
+        _recursive_add_option_filters(question, question['options'], question_filters)
+
+    return question_filters
+
+
+def _recursive_add_option_filters(question, options_list, filters_list):
+    for option in options_list:
+        if not option.get('filter_ignore'):
+            value = _get_filter_value_from_question_option(option)
+            presented_filter = {
+                'label': option.get('filter_label') or option['label'],
+                'name': question['id'],
+                'id': '{}-{}'.format(
+                    question['id'],
+                    value.replace(' ', '-')),
+                'value': value,
+            }
+            if option.get('options'):
+                presented_filter['children'] = []
+                _recursive_add_option_filters(question, option.get('options', []), presented_filter['children'])
+
+            filters_list.append(presented_filter)
+
+
+def generate_filter(framework_slug, lot_slug):
+    loader = ContentLoader('./')
+
+    loader.load_manifest(framework_slug, 'services', 'services_search_filters')
+    manifest = loader.get_manifest(framework_slug, 'services_search_filters')
+
+    return _filters_for_lot(
+        lot_slug,
+        manifest,
+        all_lots=[lot_slug for _lot_name, lot_slug in G_CLOUD_LOTS]
+    )
+
+
+def generate_filter_todir(dir_path, framework_slug, lot_slug):
+    start_t = time.perf_counter()
+    schema = generate_filter(framework_slug, lot_slug)
+
+    with open(os.path.join(dir_path, 'filters-{}-{}.json'.format(framework_slug, lot_slug)), 'w') as f:
+        json.dump(schema, f, sort_keys=True, indent=2, separators=(',', ': '))
+        f.write(os.linesep)
+
+    end_t = time.perf_counter()
+
+    return ((framework_slug, lot_slug), end_t - start_t)

package/scripts/generate-filter-schemas.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+"""Generate filters JSON schemas from the frameworks questions content.
+
+Usage:
+    generate-filters-schemas.py --output-path=<output_path> [--framework=<framework_slug>] [--lot=<lot_slug>]
+
+"""
+import os
+import sys
+sys.path.insert(0, '.')
+
+from docopt import docopt
+
+from multiprocessing import Pool
+from functools import partial
+
+from schema_generator.filter import generate_filter_todir, SCHEMAS
+
+
+if __name__ == '__main__':
+    arguments = docopt(__doc__)
+    OUTPUT_DIR = arguments['--output-path']
+    FRAMEWORK_SLUG = arguments.get('--framework')
+    LOT_SLUG = arguments.get('--lot')
+    if not os.path.exists(OUTPUT_DIR):
+        print("Creating {} directory".format(OUTPUT_DIR))
+        os.makedirs(OUTPUT_DIR)
+
+    schemas = SCHEMAS
+
+    if FRAMEWORK_SLUG:
+        schemas = [schema for schema in schemas if schema[0] == FRAMEWORK_SLUG]
+
+    if LOT_SLUG:
+        schemas = [schema for schema in schemas if schema[1] == LOT_SLUG]
+
+    with Pool(processes=5) as pool:
+        results = pool.starmap(
+            partial(
+                generate_filter_todir,
+                OUTPUT_DIR,
+            ),
+            schemas
+        )
+
+        for schema, duration in results:
+            print(f'{schema} completed in {duration:.2f}s')

package/tests/test_content.py

@@ -137,9 +137,7 @@ def test_dependent_follow_ups(framework, question_set, manifest):
     loaded_content = content.get_manifest(framework, manifest)
 
     questions_with_dependent_follow_up = {}
-    questions_with_dependent_follow_up_followup = {}
     expected_questions_with_dependent_follow_up = {}
-    expected_questions_with_dependent_follow_up_followup = {}
 
     for section in loaded_content.sections:
         for section_question in section.questions:
@@ -151,25 +149,65 @@ def test_dependent_follow_ups(framework, question_set, manifest):
             for question in questions:
                 if question.get("dependent_follow_up"):
                     questions_with_dependent_follow_up[question.id] = question.get("dependent_follow_up")
-                if question.get("dependent_follow_up_followup"):
-                    questions_with_dependent_follow_up_followup[question.id] = \
-                        question.get("dependent_follow_up_followup")
 
                 for question_id, question_values in question.get("depends_on_follow_up", {}).items():
                     if question_id not in expected_questions_with_dependent_follow_up:
                         expected_questions_with_dependent_follow_up[question_id] = {}
 
+                    for dependent_follow_up_id in question.get("dependent_follow_up", {}).keys():
+                        expected_questions_with_dependent_follow_up[
+                            question_id
+                        ][
+                            dependent_follow_up_id
+                        ] = question_values
+
                     for field in question.form_fields:
                         expected_questions_with_dependent_follow_up[question_id][field] = question_values
 
-                    for fllowup_question_id in question.get('followup', {}).keys():
+    assert questions_with_dependent_follow_up == expected_questions_with_dependent_follow_up
+
+
+@pytest.mark.parametrize("framework, question_set, manifest", get_manifests())
+def test_dependent_follow_up_followup(framework, question_set, manifest):
+    content.load_manifest(framework, question_set, manifest)
+    loaded_content = content.get_manifest(framework, manifest)
+
+    questions_with_dependent_follow_up_followup = {}
+    expected_questions_with_dependent_follow_up_followup = {}
+
+    for section in loaded_content.sections:
+        for section_question in section.questions:
+            if isinstance(section_question, Multiquestion):
+                questions = section_question.get_questions_flat()
+            else:
+                questions = [section_question]
+
+            for question in questions:
+                if question.get("dependent_follow_up_followup"):
+                    questions_with_dependent_follow_up_followup[question.id] = \
+                        question.get("dependent_follow_up_followup")
+
+                for question_id, question_values in question.get("depends_on_follow_up", {}).items():
+                    for followup_question_id in question.get('followup', {}).keys():
                         if question_id not in expected_questions_with_dependent_follow_up_followup:
                             expected_questions_with_dependent_follow_up_followup[question_id] = {}
 
-                        expected_questions_with_dependent_follow_up_followup[question_id][fllowup_question_id] = \
-                            question_values
+                        expected_questions_with_dependent_follow_up_followup[
+                            question_id
+                        ][
+                            followup_question_id
+                        ] = question_values
+
+                    for dependent_follow_up_followup_id in question.get("dependent_follow_up_followup", {}).keys():
+                        if question_id not in expected_questions_with_dependent_follow_up_followup:
+                            expected_questions_with_dependent_follow_up_followup[question_id] = {}
+
+                        expected_questions_with_dependent_follow_up_followup[
+                            question_id
+                        ][
+                            dependent_follow_up_followup_id
+                        ] = question_values
 
-    assert questions_with_dependent_follow_up == expected_questions_with_dependent_follow_up
     assert questions_with_dependent_follow_up_followup == expected_questions_with_dependent_follow_up_followup