kot 0.0.2 → 0.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: b3d7be21381f93bf3481b87a3167afeb00bc85101113fb55031d25d169434d56
4
- data.tar.gz: 57e6c62c8bf39ff09f63db2c9c70a4f3c3727ade7e216ea1cc26d7a8a42dd67f
3
+ metadata.gz: 4df58ff0365bfa26e18adba52d56d07b2ba6e1648a7672e30e36074086a59b86
4
+ data.tar.gz: 277e71849654efe4c68d6e0994195c03bfcc4758d5e768fcca3a363e56c54c01
5
5
  SHA512:
6
- metadata.gz: dfade4c0dda38f2ad2d8182abacecb4686088c6323a6a0c044507ba921aebe8fb552b3b92eb8c7a0f24a0d62d21762ce623de852d30a38b069c0cd0b42a9354e
7
- data.tar.gz: a59fbf2efccc4843e456c8a5093e263030411250d100f42163425f4b22a6b3ce778acf94a69c1d67e066f95c8dd5b0e5ec75a2db4eadd7d3eae0e32c65d2cfa8
6
+ metadata.gz: 782b27fe171ca5ed98fde2c0f83115bfc94e3272c25132779edff79f0955bff817238f53b4079bb122b563274054cd00d7399b967783abe2ad533ce8ea026642
7
+ data.tar.gz: 24ab8c8ae3af9e2bbf3bf02de19e2fbcdcbdcf552f840bb835c1bbe91cd3f4bbe27adfba0d26a0542f12c6e56245dac8e40d9171a71815d014bc8882b3015bab
data/lib/kot.rb CHANGED
@@ -1,4 +1,5 @@
1
1
  require('kot/item_response_theory')
2
+ require('kot/item4pl')
2
3
  require('kot/hill_climbing_estimator')
3
4
  require('kot/randomesque_selector')
4
- require('kot/test')
5
+ require('kot/test')
@@ -1,54 +1,53 @@
1
1
  module Kot
2
-
3
2
  class HillClimbingEstimator
4
3
 
5
-
6
-
4
+ # Estimates theta when all responses are true or all are false, based on Dodd, 1990.
7
5
  # "The variable stepsize changed the 0 estimate by half the distance to the appropriate ... value in the item pool."
8
- def dodd(est_theta:0.0, items:[], last_response:[])
6
+ def dodd(est_theta: 0.0, items: [], last_response: [])
9
7
  max_b = items.map(&:b).max
10
8
  min_b = items.map(&:b).min
11
9
 
12
10
  last_response ? est_theta + ((max_b - est_theta) / 2.0) : est_theta - ((est_theta - min_b) / 2.0)
13
11
  end
14
12
 
15
-
13
+ # Performs a single iteration of the hill climb, starting from one bound towards the other.
16
14
  def estimate_iteration(best_theta, max_ll, lower_bound, upper_bound, responses, items)
17
- step_size = (upper_bound - lower_bound) / 10
18
-
19
- case step_size <=> 0
20
- when 1
21
- intervals = (lower_bound..upper_bound).step(step_size).each
22
- when -1
23
- intervals = (upper_bound..lower_bound).step(step_size.abs).reverse_each
24
- when 0
25
- intervals = []
26
- end
27
-
28
- intervals.each do |ii|
15
+ step_size = (upper_bound - lower_bound) / 10
16
+
17
+ case step_size <=> 0
18
+ when 1
19
+ intervals = (lower_bound..upper_bound).step(step_size).each
20
+ when -1
21
+ intervals = (upper_bound..lower_bound).step(step_size.abs).reverse_each
22
+ when 0
23
+ intervals = []
24
+ end
29
25
 
30
- ll = ItemResponseTheory.log_likelihood(ii, responses, items)
26
+ intervals.each do |ii|
31
27
 
32
- if ll > max_ll
33
- max_ll = ll
28
+ ll = ItemResponseTheory.log_likelihood(ii, responses, items)
34
29
 
35
- #TODO - precision-based early exit
36
- best_theta = ii
37
- else
38
- lower_bound = best_theta - step_size.abs
39
- upper_bound = ii
40
- break
41
- end
30
+ if ll > max_ll
31
+ max_ll = ll
42
32
 
33
+ #TODO: precision-based early exit
34
+ best_theta = ii
35
+ else
36
+ lower_bound = best_theta - step_size.abs
37
+ upper_bound = ii
38
+ break
43
39
  end
44
40
 
45
- return best_theta, max_ll, lower_bound, upper_bound
41
+ end
42
+
43
+ [best_theta, max_ll, lower_bound, upper_bound]
46
44
  end
47
45
 
48
- def estimate(responses:[], items:[], all_items:[], est_theta:0.0)
46
+ # Estimate theta using multiple iterations of a hill climb, falling back to {#dodd} if all responses are true or false.
47
+ def estimate(responses: [], items: [], all_items: [], est_theta: 0.0)
49
48
  if responses.uniq.count == 1
50
- raise ArgumentError.new("Responses are all #{responses.first} but missing all_items argument") if all_items.empty?
51
- return dodd(est_theta:est_theta, items:all_items, last_response:responses.last)
49
+ raise ArgumentError, "Responses are all #{responses.first} but missing all_items argument" if all_items.empty?
50
+ return dodd(est_theta: est_theta, items: all_items, last_response: responses.last)
52
51
  end
53
52
 
54
53
  lower_bound = items.map(&:b).min
@@ -59,8 +58,6 @@ module Kot
59
58
  best_theta = - Float::INFINITY
60
59
  max_ll = - Float::INFINITY
61
60
 
62
- old_best_theta = best_theta
63
-
64
61
  10.times do
65
62
  best_theta, max_ll, lower_bound, upper_bound =
66
63
  estimate_iteration(best_theta, max_ll, lower_bound, upper_bound, responses, items)
@@ -68,9 +65,9 @@ module Kot
68
65
  break if lower_bound == upper_bound
69
66
  end
70
67
 
71
- return best_theta
68
+ best_theta
72
69
  end
73
70
 
74
71
  end
75
72
 
76
- end
73
+ end
@@ -0,0 +1,93 @@
1
+ module Kot
2
+
3
+ # An example of an Item class. You probably don't want to inherit from this,
4
+ # but instead to include {ItemResponseTheory} in a class of your own that
5
+ # provides {#a}, {#b}, {#c} and {#d}. See those attribute definitions for
6
+ # more information about what each parameter means in the context of
7
+ # correct/incorrect tests of ability.
8
+ #
9
+ # This class is useful for simulating different CAT setups and is used
10
+ # for some of the specification tests of this library.
11
+ class Item4PL
12
+ include Kot::ItemResponseTheory
13
+
14
+ attr_reader :a, :b, :c, :d
15
+
16
+ # @!attribute [r] a
17
+ # Discrimination ability of the item.
18
+ # This describe the maximum slope of the item's ICC, at the point given by {#b},
19
+ # and so how sharply the item distinguishes between those with ability below and above that point.
20
+ #
21
+ # An item with an _a_ of zero would have an entirely flat ICC (and so be completely useless),
22
+ # while an item with an infinitely high _a_ would perfectly discriminate such that anyone with an ability
23
+ # of *less* *than* _b_ would have _c_ probability of getting the answer right and anyone with an ability
24
+ # *greater* *than* _b_ would have _d_ probability of getting the answer right.
25
+ #
26
+ # Usually set to 1.0 for 1PL models.
27
+ # @return [Float] Discrimination
28
+
29
+ # @!attribute [r] b
30
+ # Difficulty of the item.
31
+ # This describes the midpoint of the item's ICC, where P(_b_)=0.5,
32
+ # and so the point at which half of those with a _theta_ equal to _b_ will answer
33
+ # the item correctly and half will answer it incorrectly.
34
+ #
35
+ # _b_ is the parameter required by any model, and across the bank of items usually
36
+ # will range over the distribution of test-takers' ability.
37
+ # (Often the distribution of ability is conceived as N(0,1).)
38
+ #
39
+ # @return [Float] Difficulty
40
+
41
+ # @!attribute [r] c
42
+ # Likelihood of guesing the item.
43
+ # This describes the lower asymptote of the item's ICC,
44
+ # and so the lowest probability of answering correctly regardless of ability.
45
+ #
46
+ # For example, a multiple choice test item with 5 possible answers,
47
+ # one of which is correct, might have a _c_ of 1/5 or 0.2 .
48
+ #
49
+ # Usually set to 0.0 for 1-2PL models.
50
+ # @return [Float] Guessing
51
+
52
+ # @!attribute [r] d
53
+ # Maximum likelihood of answering correctly.
54
+ # This describes the upper asymptote of the item's ICC,
55
+ # and so the highest probability of answering correctly regardless of ability.
56
+ #
57
+ # This parameter is more common in contexts outside of testing ability
58
+ # such as cognitive and clinical measures.
59
+ #
60
+ # Usually set to 1.0 for 1-3PL models.
61
+ # @return [Float] Insurmountable difficulty
62
+
63
+ def self.[](*arr)
64
+ arr.map { |a| Item4PL.new(a) }
65
+ end
66
+
67
+ # @see https://stackoverflow.com/questions/5825680/code-to-generate-gaussian-normally-distributed-random-numbers-in-ruby
68
+ def self.gaussian(mean = 0.0, stddev = 1.0, rand = lambda{ Kernel.rand } )
69
+ theta = 2 * Math::PI * rand.call
70
+ rho = Math.sqrt(-2 * Math.log(1 - rand.call))
71
+ scale = stddev * rho
72
+ mean + scale * Math.cos(theta)
73
+ end
74
+
75
+ # @return [Item4PL] a 1PL Item4PL with a {#b} chosen randomly from N(0,1)
76
+ # @return [Array] an array of 1PL Item4PLs with {#b}s chosen randomly from N(0,1)
77
+ def self.generate(n = nil)
78
+ return Item4PL.new(b:gaussian()) if n.nil?
79
+ Array.new(n) { generate }
80
+ end
81
+
82
+ def initialize(a: 1.0, b: 0.0, c: 0.0, d: 1.0)
83
+ @a = a
84
+ @b = b
85
+ @c = c
86
+ @d = d
87
+ end
88
+
89
+ def to_s
90
+ "<Item4PL a:#{a} b:#{b} c:#{c} d#{d} >"
91
+ end
92
+ end
93
+ end
@@ -1,53 +1,61 @@
1
1
  module Kot
2
2
 
3
- # Requires a, b, c, d
3
+ # Include this module into a class to give various IRT statistics for individual items.
4
+ # Including classes are expected to respond to #a, #b, #c and #d ; see the example {Item4PL} class for more information.
5
+ #
6
+ # Class methods for this module provide IRT statistics for a set of items,
7
+ # given an individual's estimated theta (and sometimes their responses to those items).
4
8
  module ItemResponseTheory
5
9
 
6
- #
7
- # Module methods for statistics based on estimated theta, items and responses
8
- #
9
-
10
+ # @param est_theta [Float] an estimate of an individual's ability
11
+ # @param responses [Array<TrueClass, FalseClass>] the responses given by an individual to _items_
12
+ # @param items [Array<ItemResponseTheory>] items that an individual has responded to
13
+ # @return [Float]
10
14
  def self.log_likelihood(est_theta, responses, items)
11
- ps = items.map {|i| i.icc(est_theta) }
12
- ls = ps.each_with_index.map {|e,i| responses[i] ? Math.log(e) : Math.log(1.0 - e)} #TODO: Polychotomous
15
+ ps = items.map { |i| i.icc(est_theta) }
16
+ ls = ps.each_with_index.map { |e, i| responses[i] ? Math.log(e) : Math.log(1.0 - e) }
17
+ # TODO: Polychotomous
13
18
  ls.inject(:+)
14
19
  end
15
20
 
21
+ # @param est_theta [Float] an estimate of an individual's ability
22
+ # @param items [Array<ItemResponseTheory>] items that an individual has responded to
23
+ # @return [Float]
16
24
  def self.test_info(est_theta, items)
17
- items.map {|i| i.inf(est_theta) }.inject(:+)
25
+ items.map { |i| i.inf(est_theta) }.inject(:+)
18
26
  end
19
27
 
28
+ # @param est_theta [Float] an estimate of an individual's ability
29
+ # @param items [Array<ItemResponseTheory>] items that an individual has responded to
30
+ # @return [Float]
20
31
  def self.var(est_theta, items)
21
- 1.0/test_info(est_theta, items)
32
+ 1.0 / test_info(est_theta, items)
22
33
  end
23
34
 
35
+ # @param est_theta [Float] an estimate of an individual's ability
36
+ # @param items [Array<ItemResponseTheory>] items that an individual has responded to
37
+ # @return [Float] standard error of estimation
24
38
  def self.see(est_theta, items)
25
39
  Math.sqrt(var(est_theta, items))
26
40
  end
27
41
 
28
-
29
- #
30
- # Methods intended for inclusion into Item-classes
31
- #
32
-
33
- def icc_component(theta)
34
- Math.exp(-a * (theta - b))
35
- end
36
-
37
42
  # Item characteristic curve
43
+ # @return [Float] the probability of someone with ability _theta_ of answering this item correctly
38
44
  def icc(theta)
39
- c + ((d - c) / (1.0 + icc_component(theta)))
45
+ icc_component = Math.exp(-a * (theta - b))
46
+ c + ((d - c) / (1.0 + icc_component))
40
47
  end
41
48
 
42
- # Information value of an item
49
+ # Item information function
50
+ # @return [Float] a measure of the information that would be provided by a response to this item given a prior _theta_
43
51
  def inf(theta)
44
52
  vp = icc(theta)
45
53
 
46
54
  top = (a ** 2) * ((vp - c) ** 2) * ((d - vp) ** 2)
47
55
  bottom = ((d - c) ** 2) * vp * (1.0 - vp)
48
56
 
49
- top/bottom
57
+ top / bottom
50
58
  end
51
59
 
52
60
  end
53
- end
61
+ end
@@ -7,9 +7,9 @@ module Kot
7
7
  end
8
8
 
9
9
  def select(est_theta, possible_items)
10
-
11
- possible_items.sort_by {|i| - i.inf(est_theta)}.slice(0, @bin_size).sample
10
+ possible_items.sort_by { |i| - i.inf(est_theta) }.
11
+ slice(0, @bin_size).sample
12
12
  end
13
13
 
14
14
  end
15
- end
15
+ end
data/lib/kot/test.rb CHANGED
@@ -1,6 +1,5 @@
1
1
  module Kot
2
2
 
3
-
4
3
  class Test
5
4
 
6
5
  attr_reader :est_theta
@@ -15,21 +14,24 @@ module Kot
15
14
  @asked_items = []
16
15
  end
17
16
 
18
-
17
+ # Get the standard error of estimation for the test so far.
19
18
  def see
20
19
  return Float::INFINITY if @asked_items.empty?
21
20
  ItemResponseTheory.see(@est_theta, @asked_items)
22
21
  end
23
22
 
23
+ # Update the estimated theta for the test so far.
24
24
  def update_est_theta
25
- @est_theta = @estimator.estimate(est_theta: @est_theta, responses:@responses, items:@asked_items, all_items:@item_bank)
25
+ @est_theta = @estimator.estimate(est_theta: @est_theta, responses: @responses, items: @asked_items, all_items: @item_bank)
26
26
  end
27
27
 
28
+ # Ask the selector for a new item from the item bank.
28
29
  def next_item
29
30
  possible_items = @item_bank - @asked_items
30
31
  @selector.select(@est_theta, possible_items)
31
32
  end
32
33
 
34
+ # Add a response for a given item.
33
35
  def respond(response, item)
34
36
  @responses << response
35
37
  @asked_items << item
@@ -38,4 +40,4 @@ module Kot
38
40
 
39
41
  end
40
42
 
41
- end
43
+ end
data/lib/kot/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module Kot
2
- VERSION = '0.0.2'.freeze
2
+ VERSION = '0.0.3'.freeze
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: kot
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.2
4
+ version: 0.0.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Adam Watkins
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-08-25 00:00:00.000000000 Z
11
+ date: 2018-10-03 00:00:00.000000000 Z
12
12
  dependencies: []
13
13
  description: " Kot is a basic toolkit for getting started with computerised adaptive
14
14
  testing (CAT). It includes a module to calculate item response theory (IRT) statistics
@@ -22,6 +22,7 @@ extra_rdoc_files: []
22
22
  files:
23
23
  - lib/kot.rb
24
24
  - lib/kot/hill_climbing_estimator.rb
25
+ - lib/kot/item4pl.rb
25
26
  - lib/kot/item_response_theory.rb
26
27
  - lib/kot/randomesque_selector.rb
27
28
  - lib/kot/test.rb